commit b3d162d6bfe82d84d55edd58016347a420732fe2 (tree) parent ef050483dff9f4fe57107b4c3ddba9a2692bdef4 Author: Andrew Kelley <andrew@ziglang.org> Date: Wed, 18 Feb 2026 14:09:55 -0800 build maker: rename files to match type Diffstat:
14 files changed, 2455 insertions(+), 2455 deletions(-)
diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig @@ -0,0 +1,1848 @@ +const Maker = @This(); +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const Configuration = std.Build.Configuration; +const File = std.Io.File; +const Io = std.Io; +const Path = std.Build.Cache.Path; +const Writer = std.Io.Writer; +const assert = std.debug.assert; +const fatal = std.process.fatal; +const fmt = std.fmt; +const log = std.log; +const mem = std.mem; +const process = std.process; + +const Fuzz = @import("Maker/Fuzz.zig"); +const Graph = @import("Maker/Graph.zig"); +const Step = @import("Maker/Step.zig"); +const Watch = @import("Maker/Watch.zig"); +const WebServer = @import("Maker/WebServer.zig"); + +pub const std_options: std.Options = .{ + .side_channels_mitigations = .none, + .http_disable_tls = true, +}; + +gpa: Allocator, +graph: *Graph, +install_paths: InstallPaths, +scanned_config: *const ScannedConfig, +steps: []Step, + +available_rss: usize, +max_rss_is_default: bool, +max_rss_mutex: Io.Mutex, +skip_oom_steps: bool, +unit_test_timeout_ns: ?u64, +watch: bool, +web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, +/// Allocated into `gpa`. +memory_blocked_steps: std.ArrayList(Configuration.Step.Index), +/// Allocated into `gpa`. +step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + +error_style: ErrorStyle, +multiline_errors: MultilineErrors, +summary: Summary, + +pub fn main(init: process.Init.Minimal) !void { + // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not + // always the case. So, we do need a true gpa for some things. + var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); + defer _ = safe_gpa_state.deinit(); + const gpa = safe_gpa_state.allocator(); + + var threaded: std.Io.Threaded = .init(gpa, .{ + .environ = init.environ, + .argv0 = .init(init.args), + }); + defer threaded.deinit(); + const io = threaded.io(); + + // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. + var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + const args = try init.args.toSlice(arena); + + // skip my own exe name + var arg_idx: usize = 1; + + const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); + const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); + const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); + const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); + const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); + const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); + + const cwd: Io.Dir = .cwd(); + + const zig_lib_directory: Cache.Directory = .{ + .path = zig_lib_dir, + .handle = try cwd.openDir(io, zig_lib_dir, .{}), + }; + + const build_root_directory: Cache.Directory = .{ + .path = build_root, + .handle = try cwd.openDir(io, build_root, .{}), + }; + + const local_cache_directory: Cache.Directory = .{ + .path = local_cache_root, + .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), + }; + + const global_cache_directory: Cache.Directory = .{ + .path = global_cache_root, + .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + }; + + var graph: Graph = .{ + .io = io, + .arena = arena, + .cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), + .cwd = try process.currentPathAlloc(io, arena), + }, + .zig_exe = zig_exe, + .environ_map = try init.environ.createMap(arena), + .global_cache_root = global_cache_directory, + .zig_lib_directory = zig_lib_directory, + }; + + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(build_root_directory); + graph.cache.addPrefix(local_cache_directory); + graph.cache.addPrefix(global_cache_directory); + graph.cache.hash.addBytes(builtin.zig_version_string); + + var step_names: std.ArrayList([]const u8) = .empty; + var debug_log_scopes: std.ArrayList([]const u8) = .empty; + var help_menu = false; + var steps_menu = false; + var print_configuration = false; + var override_install_prefix: ?[]const u8 = null; + var override_lib_dir: ?[]const u8 = null; + var override_bin_dir: ?[]const u8 = null; + var override_include_dir: ?[]const u8 = null; + var error_style: ErrorStyle = .verbose; + var multiline_errors: MultilineErrors = .indent; + var summary: ?Summary = null; + var max_rss: u64 = 0; + var skip_oom_steps = false; + var test_timeout_ns: ?u64 = null; + var color: Color = .auto; + var watch = false; + var fuzz: ?Fuzz.Mode = null; + var debounce_interval_ms: u16 = 50; + var webui_listen: ?Io.net.IpAddress = null; + var verbose = false; + var sysroot: ?[]const u8 = null; + var search_prefixes: std.ArrayList([]const u8) = .empty; + var libc_file: ?[]const u8 = null; + var debug_pkg_config: bool = false; + // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, + // this will be the directory $glibc-build-dir/install/glibcs + // Given the example of the aarch64 target, this is the directory + // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. + // Also works for dynamic musl. + var libc_runtimes_dir: ?[]const u8 = null; + var enable_wine = false; + var enable_qemu = false; + var enable_wasmtime = false; + var enable_darling = false; + var enable_rosetta = false; + var reference_trace: ?u32 = null; + var run_args: ?[]const []const u8 = null; + + if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + error_style = style; + } + } + + if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + multiline_errors = style; + } + } + + while (nextArg(args, &arg_idx)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + help_menu = true; + } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { + steps_menu = true; + } else if (mem.eql(u8, arg, "--print-configuration")) { + print_configuration = true; + } else if (mem.eql(u8, arg, "--verbose")) { + verbose = true; + } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { + override_install_prefix = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { + override_lib_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { + override_bin_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--prefix-include-dir")) { + override_include_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--sysroot")) { + sysroot = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--maxrss")) { + const max_rss_text = nextArgOrFatal(args, &arg_idx); + max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| + fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err }); + } else if (mem.eql(u8, arg, "--skip-oom-steps")) { + skip_oom_steps = true; + } else if (mem.eql(u8, arg, "--test-timeout")) { + const units: []const struct { []const u8, u64 } = &.{ + .{ "ns", 1 }, + .{ "nanosecond", 1 }, + .{ "us", std.time.ns_per_us }, + .{ "microsecond", std.time.ns_per_us }, + .{ "ms", std.time.ns_per_ms }, + .{ "millisecond", std.time.ns_per_ms }, + .{ "s", std.time.ns_per_s }, + .{ "second", std.time.ns_per_s }, + .{ "m", std.time.ns_per_min }, + .{ "minute", std.time.ns_per_min }, + .{ "h", std.time.ns_per_hour }, + .{ "hour", std.time.ns_per_hour }, + }; + const timeout_str = nextArgOrFatal(args, &arg_idx); + const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( + "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", + .{timeout_str}, + ); + const num_str = timeout_str[0 .. num_end_idx + 1]; + const unit_str = timeout_str[num_end_idx + 1 ..]; + const unit_factor: f64 = for (units) |unit_and_factor| { + if (std.mem.eql(u8, unit_str, unit_and_factor[0])) { + break @floatFromInt(unit_and_factor[1]); + } + } else fatal( + "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", + .{ timeout_str, unit_str }, + ); + const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( + "invalid timeout '{s}': invalid number '{s}' ({t})", + .{ timeout_str, num_str, err }, + ); + test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); + } else if (mem.eql(u8, arg, "--search-prefix")) { + try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); + } else if (mem.eql(u8, arg, "--libc")) { + libc_file = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--color")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); + color = std.meta.stringToEnum(Color, next_arg) orelse { + fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--error-style")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--multiline-errors")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected style after '{s}'", .{arg}); + multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); + }; + } else if (mem.eql(u8, arg, "--summary")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); + summary = std.meta.stringToEnum(Summary, next_arg) orelse { + fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ + arg, next_arg, + }); + }; + } else if (mem.eql(u8, arg, "--seed")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u32 after '{s}'", .{arg}); + graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { + fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err }); + }; + } else if (mem.eql(u8, arg, "--debounce")) { + const next_arg = nextArg(args, &arg_idx) orelse + fatalWithHint("expected u16 after '{s}'", .{arg}); + debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { + fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ + next_arg, err, + }); + }; + } else if (mem.eql(u8, arg, "--webui")) { + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; + } else if (mem.startsWith(u8, arg, "--webui=")) { + const addr_str = arg["--webui=".len..]; + if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); + webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { + fatal("invalid web UI address '{s}': {t}", .{ addr_str, err }); + }; + } else if (mem.eql(u8, arg, "--debug-log")) { + const next_arg = nextArgOrFatal(args, &arg_idx); + try debug_log_scopes.append(arena, next_arg); + } else if (mem.eql(u8, arg, "--debug-pkg-config")) { + debug_pkg_config = true; + } else if (mem.eql(u8, arg, "--debug-rt")) { + graph.debug_compiler_runtime_libs = .Debug; + } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { + graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + fatal("unrecognized optimization mode: {s}", .{rest}); + } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { + // --glibc-runtimes was the old name of the flag; kept for compatibility for now. + libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); + } else if (mem.eql(u8, arg, "--watch")) { + watch = true; + } else if (mem.eql(u8, arg, "--time-report")) { + graph.time_report = true; + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; + } else if (mem.eql(u8, arg, "--fuzz")) { + fuzz = .{ .forever = undefined }; + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; + } else if (mem.startsWith(u8, arg, "--fuzz=")) { + const value = arg["--fuzz=".len..]; + if (value.len == 0) fatal("missing argument to --fuzz", .{}); + + const unit: u8 = value[value.len - 1]; + const digits = switch (unit) { + '0'...'9' => value, + 'K', 'M', 'G' => value[0 .. value.len - 1], + else => fatal( + "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", + .{}, + ), + }; + + const amount = std.fmt.parseInt(u64, digits, 10) catch { + fatal( + "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", + .{}, + ); + }; + + const normalized_amount = std.math.mul(u64, amount, switch (unit) { + else => unreachable, + '0'...'9' => 1, + 'K' => 1000, + 'M' => 1_000_000, + 'G' => 1_000_000_000, + }) catch fatal("fuzzing limit amount overflows u64", .{}); + + fuzz = .{ + .limit = .{ + .amount = normalized_amount, + }, + }; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; + } else if (mem.eql(u8, arg, "-fwine")) { + enable_wine = true; + } else if (mem.eql(u8, arg, "-fno-wine")) { + enable_wine = false; + } else if (mem.eql(u8, arg, "-fqemu")) { + enable_qemu = true; + } else if (mem.eql(u8, arg, "-fno-qemu")) { + enable_qemu = false; + } else if (mem.eql(u8, arg, "-fwasmtime")) { + enable_wasmtime = true; + } else if (mem.eql(u8, arg, "-fno-wasmtime")) { + enable_wasmtime = false; + } else if (mem.eql(u8, arg, "-frosetta")) { + enable_rosetta = true; + } else if (mem.eql(u8, arg, "-fno-rosetta")) { + enable_rosetta = false; + } else if (mem.eql(u8, arg, "-fdarling")) { + enable_darling = true; + } else if (mem.eql(u8, arg, "-fno-darling")) { + enable_darling = false; + } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { + graph.allow_so_scripts = true; + } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { + graph.allow_so_scripts = false; + } else if (mem.eql(u8, arg, "-freference-trace")) { + reference_trace = 256; + } else if (mem.startsWith(u8, arg, "-freference-trace=")) { + const num = arg["-freference-trace=".len..]; + reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { + std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err }); + process.exit(1); + }; + } else if (mem.eql(u8, arg, "-fno-reference-trace")) { + reference_trace = null; + } else if (mem.cutPrefix(u8, arg, "-j")) |text| { + const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| + fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); + if (n < 1) fatal("number of jobs must be at least 1", .{}); + threaded.setAsyncLimit(.limited(n)); + graph.max_jobs = n; + } else if (mem.eql(u8, arg, "--")) { + run_args = argsRest(args, arg_idx); + break; + } else { + fatalWithHint("unrecognized argument: '{s}'", .{arg}); + } + } else { + try step_names.append(arena, arg); + } + } + + const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + + graph.stderr_mode = switch (color) { + .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), + .on => .escape_codes, + .off => .no_color, + }; + + const scanned_config: ScannedConfig = sc: { + const configuration = c: { + var file = cwd.openFile(io, configure_path, .{}) catch |err| + fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); + defer file.close(io); + break :c Configuration.loadFile(arena, io, file) catch |err| + fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); + }; + var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; + for (configuration.steps, 0..) |*conf_step, step_index_usize| { + const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); + const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); + if (flags.tag == .top_level) { + const name = step_index.ptr(&configuration).name.slice(&configuration); + try top_level_steps.put(arena, name, step_index); + } + } + break :sc .{ + .configuration = configuration, + .top_level_steps = top_level_steps, + }; + }; + + if (help_menu) { + var w = initStdoutWriter(io); + scanned_config.printUsage(&graph, w) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } else if (steps_menu) { + var w = initStdoutWriter(io); + scanned_config.printSteps(&graph, w) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } else if (print_configuration) { + var w = initStdoutWriter(io); + scanned_config.print(w) catch return stdout_writer_allocation.err.?; + w.flush() catch return stdout_writer_allocation.err.?; + return; + } + + if (webui_listen != null) { + if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); + if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); + } + + const main_progress_node = std.Progress.start(io, .{ + .disable_printing = (color == .off), + }); + defer main_progress_node.end(); + + const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ + .root_dir = .cwd(), + .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), + } else if (override_install_prefix) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else .{ + .root_dir = build_root_directory, + .sub_path = "zig-out", + }; + + const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "lib"); + + const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "bin"); + + const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ + .root_dir = .cwd(), + .sub_path = cwd_relative, + } else try install_prefix_path.join(arena, "include"); + + var maker: Maker = .{ + .gpa = gpa, + .graph = &graph, + .scanned_config = &scanned_config, + .install_paths = .{ + .prefix = install_prefix_path, + .lib = install_lib_path, + .bin = install_bin_path, + .include = install_include_path, + }, + .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), + + .available_rss = max_rss, + .max_rss_is_default = false, + .max_rss_mutex = .init, + .skip_oom_steps = skip_oom_steps, + .unit_test_timeout_ns = test_timeout_ns, + + .watch = watch, + .web_server = undefined, // set after `prepare` + .memory_blocked_steps = .empty, + .step_stack = .empty, + + .error_style = error_style, + .multiline_errors = multiline_errors, + .summary = summary orelse if (watch or webui_listen != null) .line else .failures, + }; + defer { + maker.memory_blocked_steps.deinit(gpa); + maker.step_stack.deinit(gpa); + } + + if (maker.available_rss == 0) { + maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); + maker.max_rss_is_default = true; + } + + maker.prepare(step_names.items) catch |err| switch (err) { + error.DependencyLoopDetected, error.InsufficientMemory => { + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(1); + }, + else => |e| return e, + }; + + var w: Watch = w: { + if (!watch) break :w undefined; + if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); + break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps); + }; + + const now = Io.Clock.Timestamp.now(io, .awake); + + maker.web_server = if (webui_listen) |listen_address| ws: { + if (builtin.single_threaded) unreachable; // `fatal` above + break :ws .init(.{ + .gpa = gpa, + .graph = &graph, + .all_steps = maker.step_stack.keys(), + .root_prog_node = main_progress_node, + .watch = watch, + .listen_address = listen_address, + .base_timestamp = now, + .configuration = &scanned_config.configuration, + }); + } else null; + + if (maker.web_server) |*ws| { + ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); + } + + rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); + }) { + if (maker.web_server) |*ws| ws.startBuild(); + + try maker.makeStepNames(step_names.items, main_progress_node, fuzz); + + if (maker.web_server) |*web_server| { + if (fuzz) |mode| if (mode != .forever) fatal( + "error: limited fuzzing is not implemented yet for --webui", + .{}, + ); + + web_server.finishBuild(.{ .fuzz = fuzz != null }); + } + + if (maker.web_server) |*ws| { + const c = &scanned_config.configuration; + assert(!watch); // fatal error after CLI parsing + while (true) switch (try ws.wait()) { + .rebuild => { + for (maker.step_stack.keys()) |step_index| { + const step = maker.stepByIndex(step_index); + step.state = .precheck_done; + const deps = step_index.ptr(c).deps.slice(c); + step.pending_deps = @intCast(deps.len); + step.reset(gpa); + } + continue :rebuild; + }, + }; + } + + // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. + if (!Watch.have_impl) unreachable; + + try w.update(gpa, maker.step_stack.keys()); + + // Wait until a file system notification arrives. Read all such events + // until the buffer is empty. Then wait for a debounce interval, resetting + // if any more events come in. After the debounce interval has passed, + // trigger a rebuild on all steps with modified inputs, as well as their + // recursive dependants. + var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; + const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ + w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()), + }) catch &caption_buf; + var debouncing_node = main_progress_node.start(caption, 0); + var in_debounce = false; + while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { + .timeout => { + assert(in_debounce); + debouncing_node.end(); + markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys()); + continue :rebuild; + }, + .dirty => if (!in_debounce) { + in_debounce = true; + debouncing_node.end(); + debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + }, + .clean => {}, + }; + } +} + +fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), + else => continue, + } + } + // Now that all dirty steps have been found, the remaining steps that + // succeeded from last run shall be marked "cached". + for (all_steps) |step_index| { + const step = &make_steps[@intFromEnum(step_index)]; + switch (step.state) { + .success => step.result_cached = true, + else => continue, + } + } +} + +fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { + var count: usize = 0; + for (all_steps) |step_index| { + const s = &make_steps[@intFromEnum(step_index)]; + count += @intFromBool(s.getZigProcess() != null); + } + return count; +} + +const InstallPaths = struct { + prefix: Path, + lib: Path, + bin: Path, + include: Path, +}; + +fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { + return &maker.steps[@intFromEnum(i)]; +} + +fn prepare(maker: *Maker, step_names: []const []const u8) !void { + const gpa = maker.gpa; + const graph = maker.graph; + const arena = graph.arena; + const seed: u32 = graph.random_seed; + const step_stack = &maker.step_stack; + const c = &maker.scanned_config.configuration; + + @memset(maker.steps, .{}); + + if (step_names.len == 0) { + try step_stack.put(gpa, c.default_step, {}); + } else { + try step_stack.ensureUnusedCapacity(gpa, step_names.len); + for (0..step_names.len) |i| { + const step_name = step_names[step_names.len - i - 1]; + const s = maker.scanned_config.top_level_steps.get(step_name) orelse { + log.info("to list available steps: zig build -l", .{}); + fatal("no such step: {s}", .{step_name}); + }; + step_stack.putAssumeCapacity(s, {}); + } + } + + const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); + + var rng = std.Random.DefaultPrng.init(seed); + const rand = rng.random(); + rand.shuffle(Configuration.Step.Index, starting_steps); + + for (starting_steps) |s| { + try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand); + } + + { + // Check that we have enough memory to complete the build. + var any_problems = false; + var max_needed: usize = 0; + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + const conf_step = step_index.ptr(c); + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss == 0) continue; + max_needed = @max(max_needed, max_rss); + if (max_rss > maker.available_rss) { + if (maker.skip_oom_steps) { + make_step.state = .skipped_oom; + for (make_step.dependants.items) |dependant| { + maker.stepByIndex(dependant).pending_deps -= 1; + } + } else { + log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ + conf_step.owner.depPrefixSlice(c), + conf_step.name.slice(c), + max_rss, + maker.available_rss, + }); + any_problems = true; + } + } + } + if (any_problems) { + if (maker.max_rss_is_default) { + std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ + max_needed, + }); + } + return error.InsufficientMemory; + } + } +} + +fn makeStepNames( + maker: *Maker, + step_names: []const []const u8, + parent_prog_node: std.Progress.Node, + fuzz: ?Fuzz.Mode, +) !void { + const graph = maker.graph; + const gpa = maker.gpa; + const io = graph.io; + const step_stack = &maker.step_stack; + const top_level_steps = &maker.scanned_config.top_level_steps; + const c = &maker.scanned_config.configuration; + + { + // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, + // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking + // a step is initial when it actually became ready due to an earlier initial step. + var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer initial_set.deinit(gpa); + try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); + for (step_stack.keys()) |step_index| { + const s = maker.stepByIndex(step_index); + if (s.state == .precheck_done and s.pending_deps == 0) { + initial_set.appendAssumeCapacity(step_index); + } + } + + const step_prog = parent_prog_node.start("steps", step_stack.count()); + defer step_prog.end(); + + var group: Io.Group = .init; + defer group.cancel(io); + // Start working on all of the initial steps... + for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog); + // ...and `makeStep` will trigger every other step when their last dependency finishes. + try group.await(io); + } + + assert(maker.memory_blocked_steps.items.len == 0); + + var test_pass_count: usize = 0; + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_crash_count: usize = 0; + var test_timeout_count: usize = 0; + + var test_count: usize = 0; + + var success_count: usize = 0; + var skipped_count: usize = 0; + var failure_count: usize = 0; + var pending_count: usize = 0; + var total_compile_errors: usize = 0; + + var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); + defer cleanup_task.await(io); + + for (step_stack.keys()) |step_index| { + const make_step = maker.stepByIndex(step_index); + test_pass_count += make_step.test_results.passCount(); + test_skip_count += make_step.test_results.skip_count; + test_fail_count += make_step.test_results.fail_count; + test_crash_count += make_step.test_results.crash_count; + test_timeout_count += make_step.test_results.timeout_count; + + test_count += make_step.test_results.test_count; + + switch (make_step.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + .dependency_failure => pending_count += 1, + .success => success_count += 1, + .skipped, .skipped_oom => skipped_count += 1, + .failure => { + failure_count += 1; + const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); + if (compile_errors_len > 0) { + total_compile_errors += compile_errors_len; + } + }, + } + } + + if (fuzz) |mode| blk: { + switch (builtin.os.tag) { + // Current implementation depends on two things that need to be ported to Windows: + // * Memory-mapping to share data between the fuzzer and build runner. + // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving + // many addresses to source locations). + .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), + else => {}, + } + if (@bitSizeOf(usize) != 64) { + // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, + // being compatible with file system's u64 return value. This is not the case + // on 32-bit platforms. + // Affects or affected by issues #5185, #22523, and #22464. + fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); + } + + switch (mode) { + .forever => break :blk, + .limit => {}, + } + + assert(mode == .limit); + var f = Fuzz.init( + gpa, + io, + step_stack.keys(), + parent_prog_node, + mode, + ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); + defer f.deinit(); + + f.start(); + try f.waitAndPrintReport(); + } + + // Every test has a state + assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); + + if (failure_count == 0) { + std.Progress.setStatus(.success); + } else { + std.Progress.setStatus(.failure); + } + + summary: { + switch (maker.summary) { + .all, .new, .line => {}, + .failures => if (failure_count == 0) break :summary, + .none => break :summary, + } + + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + const t = stderr.terminal(); + const w = &stderr.file_writer.interface; + + const total_count = success_count + failure_count + pending_count + skipped_count; + t.setColor(.cyan) catch {}; + t.setColor(.bold) catch {}; + w.writeAll("Build Summary: ") catch {}; + t.setColor(.reset) catch {}; + w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; + { + t.setColor(.dim) catch {}; + var first = true; + if (skipped_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; + first = false; + } + if (failure_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + if (test_count > 0) { + w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + t.setColor(.dim) catch {}; + var first = true; + if (test_skip_count > 0) { + w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; + first = false; + } + if (test_fail_count > 0) { + w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; + first = false; + } + if (test_crash_count > 0) { + w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; + first = false; + } + if (test_timeout_count > 0) { + w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; + first = false; + } + if (!first) w.writeByte(')') catch {}; + t.setColor(.reset) catch {}; + } + + w.writeAll("\n") catch {}; + + if (maker.summary == .line) break :summary; + + // Print a fancy tree with build results. + var step_stack_copy = try step_stack.clone(gpa); + defer step_stack_copy.deinit(gpa); + + var print_node: PrintNode = .{ .parent = null }; + if (step_names.len == 0) { + print_node.last = true; + printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; + } else { + const last_index = if (maker.summary == .all) top_level_steps.count() else blk: { + var i: usize = step_names.len; + while (i > 0) { + i -= 1; + const step_index = top_level_steps.get(step_names[i]).?; + const step = maker.stepByIndex(step_index); + const found = switch (maker.summary) { + .all, .line, .none => unreachable, + .failures => step.state != .success, + .new => !step.result_cached, + }; + if (found) break :blk i; + } + break :blk top_level_steps.count(); + }; + for (step_names, 0..) |step_name, i| { + const step_index = top_level_steps.get(step_name).?; + print_node.last = i + 1 == last_index; + printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { + error.Canceled => |e| return e, + else => {}, + }; + } + } + w.writeByte('\n') catch {}; + } + + if (maker.watch or maker.web_server != null) return; + + // Perhaps in the future there could be an Advanced Options flag such as + // --debug-build-runner-leaks which would make this code return instead of + // calling exit. + + const code: u8 = code: { + if (failure_count == 0) break :code 0; // success + if (maker.error_style.verboseContext()) break :code 1; // failure; print build command + break :code 2; // failure; do not print build command + }; + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(code); +} + +fn stepReady( + maker: *Maker, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, +) Io.Cancelable!void { + const graph = maker.graph; + const io = graph.io; + const c = &maker.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + if (max_rss != 0) { + try maker.max_rss_mutex.lock(io); + defer maker.max_rss_mutex.unlock(io); + if (maker.available_rss < max_rss) { + // Running this step right now could possibly exceed the allotted RSS. + maker.memory_blocked_steps.append(maker.gpa, step_index) catch + @panic("TODO eliminate memory allocation here"); + return; + } + maker.available_rss -= max_rss; + } + group.async(io, makeStep, .{ maker, group, step_index, root_prog_node }); +} + +/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready +/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must +/// have already subtracted this value from `maker.available_rss`. This function will release the RSS +/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked +/// steps after "make" completes for `s`. +fn makeStep( + maker: *Maker, + group: *Io.Group, + step_index: Configuration.Step.Index, + root_prog_node: std.Progress.Node, +) Io.Cancelable!void { + const graph = maker.graph; + const io = graph.io; + const gpa = maker.gpa; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const step_name = conf_step.name.slice(c); + const deps = conf_step.deps.slice(c); + const make_step = maker.stepByIndex(step_index); + + { + const step_prog_node = root_prog_node.start(step_name, 0); + defer step_prog_node.end(); + + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); + + const new_state: Step.State = for (deps) |dep_index| { + const dep_make_step = maker.stepByIndex(dep_index); + switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => break .dependency_failure, + + .success, .skipped => {}, + } + } else if (make_step.make(.{ + .progress_node = step_prog_node, + .watch = maker.watch, + .web_server = if (maker.web_server) |*ws| ws else null, + .unit_test_timeout_ns = maker.unit_test_timeout_ns, + .gpa = gpa, + })) state: { + break :state .success; + } else |err| switch (err) { + error.MakeFailed => .failure, + error.MakeSkipped => .skipped, + }; + + @atomicStore(Step.State, &make_step.state, new_state, .monotonic); + + switch (new_state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .failure, + .dependency_failure, + .skipped_oom, + => { + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); + std.Progress.setStatus(.failure_working); + }, + + .success, + .skipped, + => { + if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); + }, + } + } + + // No matter the result, we want to display error/warning messages. + if (make_step.result_error_bundle.errorMessageCount() > 0 or + make_step.result_error_msgs.items.len > 0 or + make_step.result_stderr.len > 0) + { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { + error.Canceled => |e| return e, + error.WriteFailed => switch (stderr.file_writer.err.?) { + error.Canceled => |e| return e, + else => {}, + }, + else => {}, + }; + } + + const max_rss = conf_step.max_rss.toBytes(); + if (max_rss != 0) { + var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; + defer dispatch_set.deinit(gpa); + + // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` + // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held. + { + try maker.max_rss_mutex.lock(io); + defer maker.max_rss_mutex.unlock(io); + maker.available_rss += max_rss; + dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch + @panic("TODO eliminate memory allocation here"); + while (maker.memory_blocked_steps.getLast()) |candidate_index| { + const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); + if (maker.available_rss < candidate_max_rss) break; + assert(maker.memory_blocked_steps.pop() == candidate_index); + dispatch_set.appendAssumeCapacity(candidate_index); + } + } + for (dispatch_set.items) |candidate| { + group.async(io, makeStep, .{ maker, group, candidate, root_prog_node }); + } + } + + for (make_step.dependants.items) |dependant_index| { + const dependant = maker.stepByIndex(dependant_index); + // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. + if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { + try stepReady(maker, group, dependant_index, root_prog_node); + } + } +} + +fn printTreeStep( + maker: *const Maker, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + parent_node: *PrintNode, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), +) !void { + const writer = stderr.writer; + const first = step_stack.swapRemove(step_index); + const summary = maker.summary; + const c = &maker.scanned_config.configuration; + const conf_step = step_index.ptr(c); + const make_step = maker.stepByIndex(step_index); + const skip = switch (summary) { + .none, .line => unreachable, + .all => false, + .new => make_step.result_cached, + .failures => make_step.state == .success, + }; + if (skip) return; + try printPrefix(parent_node, stderr); + + if (parent_node.parent != null) { + if (parent_node.last) { + try printChildNodePrefix(stderr); + } else { + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ + else => "+- ", + }); + } + } + + if (!first) try stderr.setColor(.dim); + + // dep_prefix omitted here because it is redundant with the tree. + try writer.writeAll(conf_step.name.slice(c)); + + const deps = conf_step.deps.slice(c); + + if (first) { + try printStepStatus(maker, step_index, stderr); + + const last_index = if (summary == .all) deps.len -| 1 else blk: { + var i: usize = deps.len; + while (i > 0) { + i -= 1; + + const dep_index = deps[i]; + const dep = maker.stepByIndex(dep_index); + const found = switch (summary) { + .all, .line, .none => unreachable, + .failures => dep.state != .success, + .new => !dep.result_cached, + }; + if (found) break :blk i; + } + break :blk deps.len -| 1; + }; + for (deps, 0..) |dep, i| { + var print_node: PrintNode = .{ + .parent = parent_node, + .last = i == last_index, + }; + try printTreeStep(maker, dep, stderr, &print_node, step_stack); + } + } else { + if (deps.len == 0) { + try writer.writeAll(" (reused)\n"); + } else { + try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); + } + try stderr.setColor(.reset); + } +} + +fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { + const s = maker.stepByIndex(step_index); + const writer = stderr.writer; + switch (s.state) { + .precheck_unstarted => unreachable, + .precheck_started => unreachable, + .precheck_done => unreachable, + + .dependency_failure => { + try stderr.setColor(.dim); + try writer.writeAll(" transitive failure\n"); + try stderr.setColor(.reset); + }, + + .success => { + try stderr.setColor(.green); + if (s.result_cached) { + try writer.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + assert(s.test_results.test_count == pass_count + s.test_results.skip_count); + try writer.print(" {d} pass", .{pass_count}); + if (s.test_results.skip_count > 0) { + try stderr.setColor(.reset); + try writer.writeAll(", "); + try stderr.setColor(.yellow); + try writer.print("{d} skip", .{s.test_results.skip_count}); + } + try stderr.setColor(.reset); + try writer.print(" ({d} total)", .{s.test_results.test_count}); + } else { + try writer.writeAll(" success"); + } + try stderr.setColor(.reset); + if (s.result_duration_ns) |ns| { + try stderr.setColor(.dim); + if (ns >= std.time.ns_per_min) { + try writer.print(" {d}m", .{ns / std.time.ns_per_min}); + } else if (ns >= std.time.ns_per_s) { + try writer.print(" {d}s", .{ns / std.time.ns_per_s}); + } else if (ns >= std.time.ns_per_ms) { + try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); + } else if (ns >= std.time.ns_per_us) { + try writer.print(" {d}us", .{ns / std.time.ns_per_us}); + } else { + try writer.print(" {d}ns", .{ns}); + } + try stderr.setColor(.reset); + } + if (s.result_peak_rss != 0) { + const rss = s.result_peak_rss; + try stderr.setColor(.dim); + if (rss >= 1000_000_000) { + try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); + } else if (rss >= 1000_000) { + try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); + } else if (rss >= 1000) { + try writer.print(" MaxRSS:{d}K", .{rss / 1000}); + } else { + try writer.print(" MaxRSS:{d}B", .{rss}); + } + try stderr.setColor(.reset); + } + try writer.writeAll("\n"); + }, + .skipped => { + try stderr.setColor(.yellow); + try writer.writeAll(" skipped\n"); + try stderr.setColor(.reset); + }, + .skipped_oom => { + const c = &maker.scanned_config.configuration; + const max_rss = step_index.ptr(c).max_rss.toBytes(); + try stderr.setColor(.yellow); + try writer.writeAll(" skipped (not enough memory)"); + try stderr.setColor(.dim); + try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ + max_rss, maker.available_rss, + }); + try stderr.setColor(.reset); + }, + .failure => { + try printStepFailure(maker.steps, step_index, stderr, false); + try stderr.setColor(.reset); + }, + } +} + +fn printStepFailure( + make_steps: []Step, + step_index: Configuration.Step.Index, + stderr: Io.Terminal, + dim: bool, +) !void { + const w = stderr.writer; + const s = &make_steps[@intFromEnum(step_index)]; + if (s.result_error_bundle.errorMessageCount() > 0) { + try stderr.setColor(.red); + try w.print(" {d} errors\n", .{ + s.result_error_bundle.errorMessageCount(), + }); + } else if (!s.test_results.isSuccess()) { + // These first values include all of the test "statuses". Every test is either passsed, + // skipped, failed, crashed, or timed out. + try stderr.setColor(.green); + try w.print(" {d} pass", .{s.test_results.passCount()}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + if (s.test_results.skip_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.yellow); + try w.print("{d} skip", .{s.test_results.skip_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + if (s.test_results.fail_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.red); + try w.print("{d} fail", .{s.test_results.fail_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + if (s.test_results.crash_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.red); + try w.print("{d} crash", .{s.test_results.crash_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + if (s.test_results.timeout_count > 0) { + try w.writeAll(", "); + try stderr.setColor(.red); + try w.print("{d} timeout", .{s.test_results.timeout_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + try w.print(" ({d} total)", .{s.test_results.test_count}); + + // Memory leaks are intentionally written after the total, because is isn't a test *status*, + // but just a flag that any tests -- even passed ones -- can have. We also use a different + // separator, so it looks like: + // 2 pass, 1 skip, 2 fail (5 total); 2 leaks + if (s.test_results.leak_count > 0) { + try w.writeAll("; "); + try stderr.setColor(.red); + try w.print("{d} leaks", .{s.test_results.leak_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + + // It's usually not helpful to know how many error logs there were because they tend to + // just come with other errors (e.g. crashes and leaks print stack traces, and clean + // failures print error traces). So only mention them if they're the only thing causing + // the failure. + const show_err_logs: bool = show: { + var alt_results = s.test_results; + alt_results.log_err_count = 0; + break :show alt_results.isSuccess(); + }; + if (show_err_logs) { + try w.writeAll("; "); + try stderr.setColor(.red); + try w.print("{d} error logs", .{s.test_results.log_err_count}); + try stderr.setColor(.reset); + if (dim) try stderr.setColor(.dim); + } + + try w.writeAll("\n"); + } else if (s.result_error_msgs.items.len > 0) { + try stderr.setColor(.red); + try w.writeAll(" failure\n"); + } else { + assert(s.result_stderr.len > 0); + try stderr.setColor(.red); + try w.writeAll(" w\n"); + } +} + +const PrintNode = struct { + parent: ?*PrintNode, + last: bool = false, +}; + +fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { + const parent = node.parent orelse return; + const writer = stderr.writer; + if (parent.parent == null) return; + try printPrefix(parent, stderr); + if (parent.last) { + try writer.writeAll(" "); + } else { + try writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ + else => "| ", + }); + } +} + +fn printChildNodePrefix(stderr: Io.Terminal) !void { + try stderr.writer.writeAll(switch (stderr.mode) { + .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ + else => "+- ", + }); +} + +/// Traverse the dependency graph depth-first and make it undirected by having +/// steps know their dependants (they only know dependencies at start). +/// Along the way, check that there is no dependency loop, and record the steps +/// in traversal order in `step_stack`. +/// Each step has its dependencies traversed in random order, this accomplishes +/// two things: +/// - `step_stack` will be in randomized-depth-first order, so the build runner +/// spawns initial steps in a random order +/// - each step's `dependants` list is also filled in a random order, so that +/// when it finishes executing in `makeStep`, it spawns next steps to run in +/// random order +fn constructGraphAndCheckForDependencyLoop( + gpa: Allocator, + c: *const Configuration, + steps: []Step, + step_index: Configuration.Step.Index, + step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), + rand: std.Random, +) error{ DependencyLoopDetected, OutOfMemory }!void { + const make_step: *Step = &steps[@intFromEnum(step_index)]; + switch (make_step.state) { + .precheck_started => { + log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); + return error.DependencyLoopDetected; + }, + .precheck_unstarted => { + make_step.state = .precheck_started; + + const step = step_index.ptr(c); + const dependencies = step.deps.slice(c); + try step_stack.ensureUnusedCapacity(gpa, dependencies.len); + + // We dupe to avoid shuffling the steps in the summary, it depends + // on dependencies' order. + const deps = try gpa.dupe(Configuration.Step.Index, dependencies); + defer gpa.free(deps); + + rand.shuffle(Configuration.Step.Index, deps); + + for (deps) |dep| { + const dep_step: *Step = &steps[@intFromEnum(dep)]; + try step_stack.put(gpa, dep, {}); + try dep_step.dependants.append(gpa, step_index); + constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { + error.DependencyLoopDetected => { + log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); + return err; + }, + else => return err, + }; + } + + make_step.state = .precheck_done; + make_step.pending_deps = @intCast(dependencies.len); + }, + .precheck_done => {}, + + // These don't happen until we actually run the step graph. + .dependency_failure => unreachable, + .success => unreachable, + .failure => unreachable, + .skipped => unreachable, + .skipped_oom => unreachable, + } +} + +pub fn printErrorMessages( + gpa: Allocator, + c: *const Configuration, + make_steps: []Step, + failing_step_index: Configuration.Step.Index, + options: std.zig.ErrorBundle.RenderOptions, + stderr: Io.Terminal, + error_style: ErrorStyle, + multiline_errors: MultilineErrors, +) !void { + const writer = stderr.writer; + if (error_style.verboseContext()) { + // Provide context for where these error messages are coming from by + // printing the corresponding Step subtree. + var step_stack: std.ArrayList(Configuration.Step.Index) = .empty; + defer step_stack.deinit(gpa); + try step_stack.append(gpa, failing_step_index); + while (true) { + const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; + if (last_step.dependants.items.len == 0) break; + try step_stack.append(gpa, last_step.dependants.items[0]); + } + + // Now, `step_stack` has the subtree that we want to print, in reverse order. + try stderr.setColor(.dim); + var indent: usize = 0; + while (step_stack.pop()) |step_index| : (indent += 1) { + if (indent > 0) { + try writer.splatByteAll(' ', (indent - 1) * 3); + try printChildNodePrefix(stderr); + } + + try writer.writeAll(step_index.ptr(c).name.slice(c)); + + if (step_index == failing_step_index) { + try printStepFailure(make_steps, step_index, stderr, true); + } else { + try writer.writeAll("\n"); + } + } + try stderr.setColor(.reset); + } else { + // Just print the failing step itself. + try stderr.setColor(.dim); + try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); + try printStepFailure(make_steps, failing_step_index, stderr, true); + try stderr.setColor(.reset); + } + + const failing_step = &make_steps[@intFromEnum(failing_step_index)]; + + if (failing_step.result_stderr.len > 0) { + try writer.writeAll(failing_step.result_stderr); + if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { + try writer.writeAll("\n"); + } + } + + try failing_step.result_error_bundle.renderToTerminal(options, stderr); + + for (failing_step.result_error_msgs.items) |msg| { + try stderr.setColor(.red); + try writer.writeAll("error:"); + try stderr.setColor(.reset); + if (std.mem.indexOfScalar(u8, msg, '\n') == null) { + try writer.print(" {s}\n", .{msg}); + } else switch (multiline_errors) { + .indent => { + var it = std.mem.splitScalar(u8, msg, '\n'); + try writer.print(" {s}\n", .{it.first()}); + while (it.next()) |line| { + try writer.print(" {s}\n", .{line}); + } + }, + .newline => try writer.print("\n{s}\n", .{msg}), + .none => try writer.print(" {s}\n", .{msg}), + } + } + + if (error_style.verboseContext()) { + if (failing_step.result_failed_command) |cmd_str| { + try stderr.setColor(.red); + try writer.writeAll("failed command: "); + try stderr.setColor(.reset); + try writer.writeAll(cmd_str); + try writer.writeByte('\n'); + } + } + + try writer.writeByte('\n'); +} + +fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { + if (idx.* >= args.len) return null; + defer idx.* += 1; + return args[idx.*]; +} + +fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { + return nextArg(args, idx) orelse { + fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); + }; +} + +fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { + const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); + if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); + const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); + return arg; +} + +fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { + if (idx >= args.len) return null; + return args[idx..]; +} + +const Color = std.zig.Color; +const ErrorStyle = enum { + verbose, + minimal, + verbose_clear, + minimal_clear, + fn verboseContext(s: ErrorStyle) bool { + return switch (s) { + .verbose, .verbose_clear => true, + .minimal, .minimal_clear => false, + }; + } + fn clearOnUpdate(s: ErrorStyle) bool { + return switch (s) { + .verbose, .minimal => false, + .verbose_clear, .minimal_clear => true, + }; + } +}; +const MultilineErrors = enum { indent, newline, none }; +const Summary = enum { all, new, failures, line, none }; + +fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { + log.info("to access the help menu: zig build -h", .{}); + fatal(f, args); +} + +fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { + for (steps) |step_index| { + if (true) @panic("TODO"); + const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; + if (wf.mode != .tmp) continue; + const path = wf.generated_directory.path orelse continue; + Io.Dir.cwd().deleteTree(io, path) catch |err| { + log.warn("failed to delete {s}: {t}", .{ path, err }); + }; + } +} + +var stdio_buffer_allocation: [256]u8 = undefined; +var stdout_writer_allocation: Io.File.Writer = undefined; + +fn initStdoutWriter(io: Io) *Writer { + stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); + return &stdout_writer_allocation.interface; +} + +const ScannedConfig = struct { + configuration: Configuration, + top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), + + fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { + const c = &sc.configuration; + var serializer: std.zon.Serializer = .{ .writer = w }; + var s = try serializer.beginStruct(.{}); + + try s.field("default_step", @intFromEnum(c.default_step), .{}); + { + var ss = try s.beginStructField("top_level_steps", .{}); + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { + try ss.field(name, @intFromEnum(step), .{}); + } + try ss.end(); + } + + try s.end(); + } + + fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + const c = &sc.configuration; + for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { + const step = step_index.ptr(c); + const decorated_name = if (step_index == c.default_step) + try fmt.allocPrint(arena, "{s} (default)", .{name}) + else + name; + const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); + const description = top_level.description.slice(c); + try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); + } + } + + fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { + const arena = graph.arena; + + try w.print( + \\Usage: {s} build [steps] [options] + \\ + \\Steps: + \\ + , .{graph.zig_exe}); + try printSteps(sc, graph, w); + try w.writeAll( + \\ + \\Project-Specific Options: + \\ + ); + + const available_options = sc.configuration.available_options; + if (available_options.len == 0) { + try w.print(" (none)\n", .{}); + } else { + for (available_options) |option| { + const name = option.name.slice(&sc.configuration); + const description = option.description.slice(&sc.configuration); + const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); + try w.print("{s:<30} {s}\n", .{ help, description }); + if (option.enum_options.slice(&sc.configuration)) |enum_options| { + const padding: [33]u8 = @splat(' '); + try w.writeAll(padding ++ "Supported Values:\n"); + for (enum_options) |enum_option_index| { + const enum_option = enum_option_index.slice(&sc.configuration); + try w.print(padding ++ " {s}\n", .{enum_option}); + } + } + } + } + + try w.writeAll( + \\ + \\System Integration Options: + \\ --search-prefix [path] Add a path to look for binaries, libraries, headers + \\ --sysroot [path] Set the system root directory (usually /) + \\ --libc [file] Provide a file which specifies libc paths + \\ + \\ --system [pkgdir] Disable package fetching; enable all integrations + \\ -fsys=[name] Enable a system integration + \\ -fno-sys=[name] Disable a system integration + \\ + \\ -fdarling, -fno-darling Integration with system-installed Darling to + \\ execute macOS programs on Linux hosts + \\ (default: no) + \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute + \\ foreign-architecture programs on Linux hosts + \\ (default: no) + \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc + \\ (e.g. glibc or musl) built for multiple foreign + \\ architectures, allowing execution of non-native + \\ programs that link with libc. + \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on + \\ ARM64 macOS hosts. (default: no) + \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to + \\ execute WASI binaries. (default: no) + \\ -fwine, -fno-wine Integration with system-installed Wine to execute + \\ Windows programs on Linux hosts. (default: no) + \\ + \\ Available System Integrations: Enabled: + \\ + ); + if (sc.configuration.system_integrations.len == 0) { + try w.writeAll(" (none) -\n"); + } else { + for (sc.configuration.system_integrations) |system_integration| { + const name = system_integration.name.slice(&sc.configuration); + const status = switch (system_integration.status) { + .disabled => "no", + .enabled => "yes", + }; + try w.print(" {s:<43} {s}\n", .{ name, status }); + } + } + + try w.writeAll( + \\ + \\General Options: + \\ -h, --help Print this help to stdout and exit + \\ -l, --list-steps Print available steps to stdout and exit + \\ + \\ -p, --prefix [path] Where to install files (default: zig-out) + \\ --prefix-lib-dir [path] Where to install libraries + \\ --prefix-exe-dir [path] Where to install executables + \\ --prefix-include-dir [path] Where to install C header files + \\ --release[=mode] Request release mode, optionally specifying a + \\ preferred optimization mode: fast, safe, small + \\ + \\ --verbose Print commands before executing them + \\ --color [auto|off|on] Enable or disable colored error messages + \\ --error-style [style] Control how build errors are printed + \\ verbose (Default) Report errors with full context + \\ minimal Report errors after summary, excluding context like command lines + \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update + \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update + \\ --multiline-errors [style] Control how multi-line error messages are printed + \\ indent (Default) Indent non-initial lines to align with initial line + \\ newline Include a leading newline so that the error message is on its own lines + \\ none Print as usual so the first line is misaligned + \\ --summary [mode] Control the printing of the build summary + \\ all Print the build summary in its entirety + \\ new Omit cached steps + \\ failures (Default if short-lived) Only print failed steps + \\ line (Default if long-lived) Only print the single-line summary + \\ none Do not print the build summary + \\ -j<N> Limit concurrent jobs (default is to use all CPU cores) + \\ --maxrss <bytes> Limit memory usage (default is to use available memory) + \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss + \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded. + \\ The timeout must include a unit: ns, us, ms, s, m, h + \\ --watch Continuously rebuild when source files are modified + \\ --debounce <ms> Delay before rebuilding after changed file detected + \\ --webui[=ip] Enable the web interface on the given IP address + \\ --fuzz[=limit] Continuously search for unit test failures with an optional + \\ limit to the max number of iterations. The argument supports + \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies + \\ '--webui' when no limit is specified. + \\ --time-report Force full rebuild and provide detailed information on + \\ compilation time of Zig source code (implies '--webui') + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation + \\ + \\Package Management Options: + \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit + \\ needed (Default) Lazy dependencies are fetched as needed + \\ all Lazy dependencies are always fetched + \\ --fork=[path] Override one or more projects from dependency tree + \\ + \\Advanced Options: + \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error + \\ -fno-reference-trace Disable reference trace + \\ -fallow-so-scripts Allows .so files to be GNU ld scripts + \\ -fno-allow-so-scripts (default) .so files must be ELF files + \\ --build-file [file] Override path to build.zig + \\ --cache-dir [path] Override path to local Zig cache directory + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --zig-lib-dir [arg] Override path to Zig lib directory + \\ --build-runner [file] Override path to build runner + \\ --seed [integer] For shuffling dependency traversal order (default: random) + \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries + \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) + \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) + \\ md5 16-byte cryptographic hash (ELF) + \\ uuid 16-byte random UUID (ELF, WASM) + \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) + \\ none (default) No build ID + \\ --debug-log [scope] Enable debugging the compiler + \\ --debug-pkg-config Fail if unknown pkg-config flags encountered + \\ --debug-rt Debug compiler runtime libraries + \\ --verbose-link Enable compiler debug output for linking + \\ --verbose-air Enable compiler debug output for Zig AIR + \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR + \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC + \\ --verbose-cimport Enable compiler debug output for C imports + \\ --verbose-cc Enable compiler debug output for C compilation + \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features + \\ + ); + } +}; diff --git a/lib/compiler/Maker/Fuzz.zig b/lib/compiler/Maker/Fuzz.zig @@ -0,0 +1,606 @@ +const Fuzz = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Build = std.Build; +const Cache = std.Build.Cache; +const Coverage = std.debug.Coverage; +const Configuration = std.Build.Configuration; +const Io = std.Io; +const abi = std.Build.abi.fuzz; +const assert = std.debug.assert; +const fatal = std.process.fatal; +const log = std.log; + +const Maker = @import("../Maker.zig"); +const WebServer = @import("WebServer.zig"); + +gpa: Allocator, +io: Io, +mode: Mode, + +/// Allocated into `gpa`. +run_steps: []const Configuration.Step.Index, + +group: Io.Group, +root_prog_node: std.Progress.Node, +prog_node: std.Progress.Node, + +/// Protects `coverage_files`. +coverage_mutex: Io.Mutex, +coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), + +queue_mutex: Io.Mutex, +queue_cond: Io.Condition, +msg_queue: std.ArrayList(Msg), + +pub const Mode = union(enum) { + forever: struct { ws: *WebServer }, + limit: Limited, + + pub const Limited = struct { + amount: u64, + }; +}; + +const Msg = union(enum) { + coverage: struct { + id: u64, + cumulative: struct { + runs: u64, + unique: u64, + coverage: u64, + }, + run: Configuration.Step.Index, + }, + entry_point: struct { + coverage_id: u64, + addr: u64, + }, +}; + +const CoverageMap = struct { + mapped_memory: []align(std.heap.page_size_min) const u8, + coverage: Coverage, + source_locations: []Coverage.SourceLocation, + /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. + entry_points: std.ArrayList(u32), + start_timestamp: i64, + start_n_runs: u64, + + fn deinit(cm: *CoverageMap, gpa: Allocator) void { + std.posix.munmap(cm.mapped_memory); + cm.coverage.deinit(gpa); + cm.* = undefined; + } +}; + +pub fn init( + gpa: Allocator, + io: Io, + all_steps: []const Configuration.Step.Index, + root_prog_node: std.Progress.Node, + mode: Mode, +) error{ OutOfMemory, Canceled }!Fuzz { + const run_steps: []const Configuration.Step.Index = steps: { + var steps: std.ArrayList(Configuration.Step.Index) = .empty; + defer steps.deinit(gpa); + const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); + defer rebuild_node.end(); + var rebuild_group: Io.Group = .init; + defer rebuild_group.cancel(io); + + for (all_steps) |step| { + if (true) @panic("TODO"); + const run = step.cast(std.Build.Step.Run) orelse continue; + if (run.producer == null) continue; + if (run.fuzz_tests.items.len == 0) continue; + try steps.append(gpa, run); + rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); + } + + if (steps.items.len == 0) fatal("no fuzz tests found", .{}); + rebuild_node.setEstimatedTotalItems(steps.items.len); + const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items); + try rebuild_group.await(io); + break :steps run_steps; + }; + errdefer gpa.free(run_steps); + + for (run_steps) |run_step_index| { + if (true) @panic("TODO"); + assert(run_step_index.fuzz_tests.items.len > 0); + if (run_step_index.rebuilt_executable == null) + fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); + } + + return .{ + .gpa = gpa, + .io = io, + .mode = mode, + .run_steps = run_steps, + .group = .init, + .root_prog_node = root_prog_node, + .prog_node = .none, + .coverage_files = .empty, + .coverage_mutex = .init, + .queue_mutex = .init, + .queue_cond = .init, + .msg_queue = .empty, + }; +} + +pub fn start(fuzz: *Fuzz) void { + const io = fuzz.io; + fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); + + if (fuzz.mode == .forever) { + // For polling messages and sending updates to subscribers. + fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err| + fatal("unable to spawn coverage task: {t}", .{err}); + } + + if (true) @panic("TODO"); + + for (fuzz.run_steps) |run| { + assert(run.rebuilt_executable != null); + fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); + } +} + +pub fn deinit(fuzz: *Fuzz) void { + const io = fuzz.io; + fuzz.group.cancel(io); + fuzz.prog_node.end(); + fuzz.gpa.free(fuzz.run_steps); +} + +fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { + rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { + const compile = run.producer.?; + log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); + }; +} + +fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { + const graph = run.step.owner.graph; + const io = graph.io; + const compile = run.producer.?; + const prog_node = parent_prog_node.start(compile.step.name, 0); + defer prog_node.end(); + + const result = compile.rebuildInFuzzMode(gpa, prog_node); + + const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; + const show_error_msgs = compile.step.result_error_msgs.items.len > 0; + const show_stderr = compile.step.result_stderr.len > 0; + + if (show_error_msgs or show_compile_errors or show_stderr) { + var buf: [256]u8 = undefined; + const stderr = try io.lockStderr(&buf, graph.stderr_mode); + defer io.unlockStderr(); + Maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + } + + const rebuilt_bin_path = result catch |err| switch (err) { + error.MakeFailed => return, + else => |other| return other, + }; + run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); +} + +fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { + const owner = run.step.owner; + const gpa = owner.allocator; + const graph = owner.graph; + const io = graph.io; + + run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { + error.MakeFailed => { + var buf: [256]u8 = undefined; + const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { + error.Canceled => return, + }; + defer io.unlockStderr(); + Maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; + return; + }, + else => { + log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); + return; + }, + }; +} + +pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { + if (true) @panic("TODO"); + assert(fuzz.mode == .forever); + + var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); + var dedup_table: DedupTable = .empty; + defer dedup_table.deinit(fuzz.gpa); + + for (fuzz.run_steps) |run_step| { + const compile_inputs = run_step.producer.?.step.inputs.table; + for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { + try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); + for (file_list.items) |sub_path| { + if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; + const joined_path = try dir_path.join(arena, sub_path); + dedup_table.putAssumeCapacity(joined_path, {}); + } + } + } + + const deduped_paths = dedup_table.keys(); + const SortContext = struct { + pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { + _ = this; + return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { + .lt => true, + .gt => false, + .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), + }; + } + }; + std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); + return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths); +} + +pub const Previous = struct { + unique_runs: usize, + entry_points: usize, + sent_source_index: bool, + pub const init: Previous = .{ + .unique_runs = 0, + .entry_points = 0, + .sent_source_index = false, + }; +}; +pub fn sendUpdate( + fuzz: *Fuzz, + socket: *std.http.Server.WebSocket, + prev: *Previous, +) !void { + const io = fuzz.io; + + try fuzz.coverage_mutex.lock(io); + defer fuzz.coverage_mutex.unlock(io); + + const coverage_maps = fuzz.coverage_files.values(); + if (coverage_maps.len == 0) return; + // TODO: handle multiple fuzz steps in the WebSocket packets + const coverage_map = &coverage_maps[0]; + const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the + // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the + // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass + // this data straight to the socket with sendfile... + const seen_pcs = cov_header.seenBits(); + const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); + const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); + { + if (!prev.sent_source_index) { + prev.sent_source_index = true; + // We need to send initial context. + const header: abi.SourceIndexHeader = .{ + .directories_len = @intCast(coverage_map.coverage.directories.entries.len), + .files_len = @intCast(coverage_map.coverage.files.entries.len), + .source_locations_len = @intCast(coverage_map.source_locations.len), + .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), + .start_timestamp = coverage_map.start_timestamp, + .start_n_runs = coverage_map.start_n_runs, + }; + var iovecs: [5][]const u8 = .{ + @ptrCast(&header), + @ptrCast(coverage_map.coverage.directories.keys()), + @ptrCast(coverage_map.coverage.files.keys()), + @ptrCast(coverage_map.source_locations), + coverage_map.coverage.string_bytes.items, + }; + try socket.writeMessageVec(&iovecs, .binary); + } + + const header: abi.CoverageUpdateHeader = .{ + .n_runs = n_runs, + .unique_runs = unique_runs, + }; + var iovecs: [2][]const u8 = .{ + @ptrCast(&header), + @ptrCast(seen_pcs), + }; + try socket.writeMessageVec(&iovecs, .binary); + + prev.unique_runs = unique_runs; + } + + if (prev.entry_points != coverage_map.entry_points.items.len) { + const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); + var iovecs: [2][]const u8 = .{ + @ptrCast(&header), + @ptrCast(coverage_map.entry_points.items), + }; + try socket.writeMessageVec(&iovecs, .binary); + + prev.entry_points = coverage_map.entry_points.items.len; + } +} + +fn coverageRun(fuzz: *Fuzz) void { + coverageRunCancelable(fuzz) catch |err| switch (err) { + error.Canceled => return, + }; +} + +fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { + const io = fuzz.io; + + try fuzz.queue_mutex.lock(io); + defer fuzz.queue_mutex.unlock(io); + + while (true) { + try fuzz.queue_cond.wait(io, &fuzz.queue_mutex); + for (fuzz.msg_queue.items) |msg| switch (msg) { + .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { + error.AlreadyReported => continue, + error.Canceled => return, + else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), + }, + .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { + error.AlreadyReported => continue, + error.Canceled => return, + else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), + }, + }; + fuzz.msg_queue.clearRetainingCapacity(); + } +} +fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { + if (true) @panic("TODO"); + assert(fuzz.mode == .forever); + const ws = fuzz.mode.forever.ws; + const gpa = fuzz.gpa; + const io = fuzz.io; + + try fuzz.coverage_mutex.lock(io); + defer fuzz.coverage_mutex.unlock(io); + + const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); + if (gop.found_existing) { + // We are fuzzing the same executable with multiple threads. + // Perhaps the same unit test; perhaps a different one. In any + // case, since the coverage file is the same, we only have to + // notice changes to that one file in order to learn coverage for + // this particular executable. + return; + } + errdefer _ = fuzz.coverage_files.pop(); + + gop.value_ptr.* = .{ + .coverage = std.debug.Coverage.init, + .mapped_memory = undefined, // populated below + .source_locations = undefined, // populated below + .entry_points = .empty, + .start_timestamp = ws.now(), + .start_n_runs = undefined, // populated below + }; + errdefer gop.value_ptr.coverage.deinit(gpa); + + const rebuilt_exe_path = run_step_index.rebuilt_executable.?; + const target = run_step_index.producer.?.rootModuleTarget(); + var debug_info = std.debug.Info.load( + gpa, + io, + rebuilt_exe_path, + &gop.value_ptr.coverage, + target.ofmt, + target.cpu.arch, + ) catch |err| { + log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ + run_step_index.step.name, rebuilt_exe_path, err, + }); + return error.AlreadyReported; + }; + defer debug_info.deinit(gpa); + + const coverage_file_path: Build.Cache.Path = .{ + .root_dir = run_step_index.step.owner.cache_root, + .sub_path = "v/" ++ std.fmt.hex(coverage_id), + }; + var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { + log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ + run_step_index.step.name, coverage_file_path, err, + }); + return error.AlreadyReported; + }; + defer coverage_file.close(io); + + const file_size = coverage_file.length(io) catch |err| { + log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); + return error.AlreadyReported; + }; + + const mapped_memory = std.posix.mmap( + null, + file_size, + .{ .READ = true }, + .{ .TYPE = .SHARED }, + coverage_file.handle, + 0, + ) catch |err| { + log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); + return error.AlreadyReported; + }; + gop.value_ptr.mapped_memory = mapped_memory; + + const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + const pcs = header.pcAddrs(); + const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); + errdefer gpa.free(source_locations); + + // Unfortunately the PCs array that LLVM gives us from the 8-bit PC + // counters feature is not sorted. + var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty; + defer sorted_pcs.deinit(gpa); + try sorted_pcs.resize(gpa, pcs.len); + @memcpy(sorted_pcs.items(.pc), pcs); + for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); + sorted_pcs.sortUnstable(struct { + addrs: []const u64, + + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.addrs[a_index] < ctx.addrs[b_index]; + } + }{ .addrs = sorted_pcs.items(.pc) }); + + debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { + log.err("failed to resolve addresses to source locations: {t}", .{err}); + return error.AlreadyReported; + }; + + for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; + gop.value_ptr.source_locations = source_locations; + gop.value_ptr.start_n_runs = header.n_runs; + + ws.notifyUpdate(); +} + +fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { + const io = fuzz.io; + + try fuzz.coverage_mutex.lock(io); + defer fuzz.coverage_mutex.unlock(io); + + const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; + const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); + const pcs = header.pcAddrs(); + + // Since this pcs list is unsorted, we must linear scan for the best index. + const index = i: { + var best: usize = 0; + for (pcs[1..], 1..) |elem_addr, i| { + if (elem_addr == addr) break :i i; + if (elem_addr > addr) continue; + if (elem_addr > pcs[best]) best = i; + } + break :i best; + }; + if (index >= pcs.len) { + log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ + addr, pcs[0], pcs[pcs.len - 1], + }); + return error.AlreadyReported; + } + if (false) { + const sl = coverage_map.source_locations[index]; + const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); + if (pcs.len == 1) { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{ + addr, file_name, sl.line, sl.column, + }); + } else if (index == 0) { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{ + addr, file_name, sl.line, sl.column, pcs[index + 1], + }); + } else if (index == pcs.len - 1) { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{ + addr, file_name, sl.line, sl.column, index, pcs[index - 1], + }); + } else { + log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ + addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], + }); + } + } + try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); +} + +pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { + if (true) @panic("TODO"); + assert(fuzz.mode == .limit); + const io = fuzz.io; + + try fuzz.group.await(io); + fuzz.group = .init; + + std.debug.print("======= FUZZING REPORT =======\n", .{}); + for (fuzz.msg_queue.items) |msg| { + if (msg != .coverage) continue; + + const cov = msg.coverage; + const coverage_file_path: std.Build.Cache.Path = .{ + .root_dir = cov.run.step.owner.cache_root, + .sub_path = "v/" ++ std.fmt.hex(cov.id), + }; + var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { + fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ + cov.run.step.name, coverage_file_path, err, + }); + }; + defer coverage_file.close(io); + + const fuzz_abi = std.Build.abi.fuzz; + var rbuf: [0x1000]u8 = undefined; + var r = coverage_file.reader(io, &rbuf); + + var header: fuzz_abi.SeenPcsHeader = undefined; + r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { + fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ + cov.run.step.name, coverage_file_path, err, + }); + }; + + if (header.pcs_len == 0) { + fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ + cov.run.step.name, coverage_file_path, + }); + } + + var seen_count: usize = 0; + const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); + for (0..chunk_count) |_| { + const seen = r.interface.takeInt(usize, .little) catch |err| { + fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ + cov.run.step.name, coverage_file_path, err, + }); + }; + seen_count += @popCount(seen); + } + + const seen_f: f64 = @floatFromInt(seen_count); + const total_f: f64 = @floatFromInt(header.pcs_len); + const ratio = seen_f / total_f; + std.debug.print( + \\Step: {s} + \\Fuzz test: "{s}" ({x}) + \\Runs: {} -> {} + \\Unique runs: {} -> {} + \\Coverage: {}/{} -> {}/{} ({:.02}%) + \\ + , .{ + cov.run.step.name, + cov.run.fuzz_tests.items[0], + cov.id, + cov.cumulative.runs, + header.n_runs, + cov.cumulative.unique, + header.unique_runs, + cov.cumulative.coverage, + header.pcs_len, + seen_count, + header.pcs_len, + ratio * 100, + }); + + std.debug.print("------------------------------\n", .{}); + } + std.debug.print( + \\Values are accumulated across multiple runs when preserving the cache. + \\============================== + \\ + , .{}); +} diff --git a/lib/compiler/maker/Graph.zig b/lib/compiler/Maker/Graph.zig diff --git a/lib/compiler/maker/Step.zig b/lib/compiler/Maker/Step.zig diff --git a/lib/compiler/maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig diff --git a/lib/compiler/maker/Step/InstallArtifact.zig b/lib/compiler/Maker/Step/InstallArtifact.zig diff --git a/lib/compiler/maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig diff --git a/lib/compiler/maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig diff --git a/lib/compiler/maker/Watch.zig b/lib/compiler/Maker/Watch.zig diff --git a/lib/compiler/maker/Watch/FsEvents.zig b/lib/compiler/Maker/Watch/FsEvents.zig diff --git a/lib/compiler/maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig diff --git a/lib/compiler/maker.zig b/lib/compiler/maker.zig @@ -1,1848 +0,0 @@ -const Maker = @This(); -const builtin = @import("builtin"); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const Configuration = std.Build.Configuration; -const File = std.Io.File; -const Io = std.Io; -const Path = std.Build.Cache.Path; -const Writer = std.Io.Writer; -const assert = std.debug.assert; -const fatal = std.process.fatal; -const fmt = std.fmt; -const log = std.log; -const mem = std.mem; -const process = std.process; - -const Fuzz = @import("maker/Fuzz.zig"); -const Graph = @import("maker/Graph.zig"); -const Step = @import("maker/Step.zig"); -const Watch = @import("maker/Watch.zig"); -const WebServer = @import("maker/WebServer.zig"); - -pub const std_options: std.Options = .{ - .side_channels_mitigations = .none, - .http_disable_tls = true, -}; - -gpa: Allocator, -graph: *Graph, -install_paths: InstallPaths, -scanned_config: *const ScannedConfig, -steps: []Step, - -available_rss: usize, -max_rss_is_default: bool, -max_rss_mutex: Io.Mutex, -skip_oom_steps: bool, -unit_test_timeout_ns: ?u64, -watch: bool, -web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, -/// Allocated into `gpa`. -memory_blocked_steps: std.ArrayList(Configuration.Step.Index), -/// Allocated into `gpa`. -step_stack: std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), - -error_style: ErrorStyle, -multiline_errors: MultilineErrors, -summary: Summary, - -pub fn main(init: process.Init.Minimal) !void { - // The build runner is often short-lived, but thanks to `--watch` and `--webui`, that's not - // always the case. So, we do need a true gpa for some things. - var safe_gpa_state: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{}); - defer _ = safe_gpa_state.deinit(); - const gpa = safe_gpa_state.allocator(); - - var threaded: std.Io.Threaded = .init(gpa, .{ - .environ = init.environ, - .argv0 = .init(init.args), - }); - defer threaded.deinit(); - const io = threaded.io(); - - // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. - var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - const args = try init.args.toSlice(arena); - - // skip my own exe name - var arg_idx: usize = 1; - - const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); - const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); - const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); - const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); - const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); - const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); - - const cwd: Io.Dir = .cwd(); - - const zig_lib_directory: Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try cwd.openDir(io, zig_lib_dir, .{}), - }; - - const build_root_directory: Cache.Directory = .{ - .path = build_root, - .handle = try cwd.openDir(io, build_root, .{}), - }; - - const local_cache_directory: Cache.Directory = .{ - .path = local_cache_root, - .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), - }; - - const global_cache_directory: Cache.Directory = .{ - .path = global_cache_root, - .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), - }; - - var graph: Graph = .{ - .io = io, - .arena = arena, - .cache = .{ - .io = io, - .gpa = gpa, - .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), - .cwd = try process.currentPathAlloc(io, arena), - }, - .zig_exe = zig_exe, - .environ_map = try init.environ.createMap(arena), - .global_cache_root = global_cache_directory, - .zig_lib_directory = zig_lib_directory, - }; - - graph.cache.addPrefix(.{ .path = null, .handle = cwd }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); - - var step_names: std.ArrayList([]const u8) = .empty; - var debug_log_scopes: std.ArrayList([]const u8) = .empty; - var help_menu = false; - var steps_menu = false; - var print_configuration = false; - var override_install_prefix: ?[]const u8 = null; - var override_lib_dir: ?[]const u8 = null; - var override_bin_dir: ?[]const u8 = null; - var override_include_dir: ?[]const u8 = null; - var error_style: ErrorStyle = .verbose; - var multiline_errors: MultilineErrors = .indent; - var summary: ?Summary = null; - var max_rss: u64 = 0; - var skip_oom_steps = false; - var test_timeout_ns: ?u64 = null; - var color: Color = .auto; - var watch = false; - var fuzz: ?Fuzz.Mode = null; - var debounce_interval_ms: u16 = 50; - var webui_listen: ?Io.net.IpAddress = null; - var verbose = false; - var sysroot: ?[]const u8 = null; - var search_prefixes: std.ArrayList([]const u8) = .empty; - var libc_file: ?[]const u8 = null; - var debug_pkg_config: bool = false; - // After following the steps in https://codeberg.org/ziglang/infra/src/branch/master/libc-update/glibc.md, - // this will be the directory $glibc-build-dir/install/glibcs - // Given the example of the aarch64 target, this is the directory - // that contains the path `aarch64-linux-gnu/lib/ld-linux-aarch64.so.1`. - // Also works for dynamic musl. - var libc_runtimes_dir: ?[]const u8 = null; - var enable_wine = false; - var enable_qemu = false; - var enable_wasmtime = false; - var enable_darling = false; - var enable_rosetta = false; - var reference_trace: ?u32 = null; - var run_args: ?[]const []const u8 = null; - - if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(ErrorStyle, str)) |style| { - error_style = style; - } - } - - if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(MultilineErrors, str)) |style| { - multiline_errors = style; - } - } - - while (nextArg(args, &arg_idx)) |arg| { - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - help_menu = true; - } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { - steps_menu = true; - } else if (mem.eql(u8, arg, "--print-configuration")) { - print_configuration = true; - } else if (mem.eql(u8, arg, "--verbose")) { - verbose = true; - } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - override_install_prefix = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - override_lib_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - override_bin_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - override_include_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--sysroot")) { - sysroot = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--maxrss")) { - const max_rss_text = nextArgOrFatal(args, &arg_idx); - max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| - fatal("invalid byte size: '{s}': {t}", .{ max_rss_text, err }); - } else if (mem.eql(u8, arg, "--skip-oom-steps")) { - skip_oom_steps = true; - } else if (mem.eql(u8, arg, "--test-timeout")) { - const units: []const struct { []const u8, u64 } = &.{ - .{ "ns", 1 }, - .{ "nanosecond", 1 }, - .{ "us", std.time.ns_per_us }, - .{ "microsecond", std.time.ns_per_us }, - .{ "ms", std.time.ns_per_ms }, - .{ "millisecond", std.time.ns_per_ms }, - .{ "s", std.time.ns_per_s }, - .{ "second", std.time.ns_per_s }, - .{ "m", std.time.ns_per_min }, - .{ "minute", std.time.ns_per_min }, - .{ "h", std.time.ns_per_hour }, - .{ "hour", std.time.ns_per_hour }, - }; - const timeout_str = nextArgOrFatal(args, &arg_idx); - const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( - "invalid timeout '{s}': expected unit (ns, us, ms, s, m, h)", - .{timeout_str}, - ); - const num_str = timeout_str[0 .. num_end_idx + 1]; - const unit_str = timeout_str[num_end_idx + 1 ..]; - const unit_factor: f64 = for (units) |unit_and_factor| { - if (std.mem.eql(u8, unit_str, unit_and_factor[0])) { - break @floatFromInt(unit_and_factor[1]); - } - } else fatal( - "invalid timeout '{s}': invalid unit '{s}' (expected ns, us, ms, s, m, h)", - .{ timeout_str, unit_str }, - ); - const num_parsed = std.fmt.parseFloat(f64, num_str) catch |err| fatal( - "invalid timeout '{s}': invalid number '{s}' ({t})", - .{ timeout_str, num_str, err }, - ); - test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); - } else if (mem.eql(u8, arg, "--search-prefix")) { - try search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); - } else if (mem.eql(u8, arg, "--libc")) { - libc_file = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [auto|on|off] after '{s}'", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after '{s}', found '{s}'", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); - error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); - }; - } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected style after '{s}'", .{arg}); - multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { - fatalWithHint("expected style after '{s}', found '{s}'", .{ arg, next_arg }); - }; - } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [all|new|failures|line|none] after '{s}'", .{arg}); - summary = std.meta.stringToEnum(Summary, next_arg) orelse { - fatalWithHint("expected [all|new|failures|line|none] after '{s}', found '{s}'", .{ - arg, next_arg, - }); - }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after '{s}'", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed '{s}' as unsigned 32-bit integer: {t}", .{ next_arg, err }); - }; - } else if (mem.eql(u8, arg, "--debounce")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u16 after '{s}'", .{arg}); - debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { - fatal("unable to parse debounce interval '{s}' as unsigned 16-bit integer: {t}\n", .{ - next_arg, err, - }); - }; - } else if (mem.eql(u8, arg, "--webui")) { - if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; - } else if (mem.startsWith(u8, arg, "--webui=")) { - const addr_str = arg["--webui=".len..]; - if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); - webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { - fatal("invalid web UI address '{s}': {t}", .{ addr_str, err }); - }; - } else if (mem.eql(u8, arg, "--debug-log")) { - const next_arg = nextArgOrFatal(args, &arg_idx); - try debug_log_scopes.append(arena, next_arg); - } else if (mem.eql(u8, arg, "--debug-pkg-config")) { - debug_pkg_config = true; - } else if (mem.eql(u8, arg, "--debug-rt")) { - graph.debug_compiler_runtime_libs = .Debug; - } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse - fatal("unrecognized optimization mode: {s}", .{rest}); - } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { - // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--watch")) { - watch = true; - } else if (mem.eql(u8, arg, "--time-report")) { - graph.time_report = true; - if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; - } else if (mem.eql(u8, arg, "--fuzz")) { - fuzz = .{ .forever = undefined }; - if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; - } else if (mem.startsWith(u8, arg, "--fuzz=")) { - const value = arg["--fuzz=".len..]; - if (value.len == 0) fatal("missing argument to --fuzz", .{}); - - const unit: u8 = value[value.len - 1]; - const digits = switch (unit) { - '0'...'9' => value, - 'K', 'M', 'G' => value[0 .. value.len - 1], - else => fatal( - "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", - .{}, - ), - }; - - const amount = std.fmt.parseInt(u64, digits, 10) catch { - fatal( - "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]", - .{}, - ); - }; - - const normalized_amount = std.math.mul(u64, amount, switch (unit) { - else => unreachable, - '0'...'9' => 1, - 'K' => 1000, - 'M' => 1_000_000, - 'G' => 1_000_000_000, - }) catch fatal("fuzzing limit amount overflows u64", .{}); - - fuzz = .{ - .limit = .{ - .amount = normalized_amount, - }, - }; - } else if (mem.eql(u8, arg, "-fincremental")) { - graph.incremental = true; - } else if (mem.eql(u8, arg, "-fno-incremental")) { - graph.incremental = false; - } else if (mem.eql(u8, arg, "-fwine")) { - enable_wine = true; - } else if (mem.eql(u8, arg, "-fno-wine")) { - enable_wine = false; - } else if (mem.eql(u8, arg, "-fqemu")) { - enable_qemu = true; - } else if (mem.eql(u8, arg, "-fno-qemu")) { - enable_qemu = false; - } else if (mem.eql(u8, arg, "-fwasmtime")) { - enable_wasmtime = true; - } else if (mem.eql(u8, arg, "-fno-wasmtime")) { - enable_wasmtime = false; - } else if (mem.eql(u8, arg, "-frosetta")) { - enable_rosetta = true; - } else if (mem.eql(u8, arg, "-fno-rosetta")) { - enable_rosetta = false; - } else if (mem.eql(u8, arg, "-fdarling")) { - enable_darling = true; - } else if (mem.eql(u8, arg, "-fno-darling")) { - enable_darling = false; - } else if (mem.eql(u8, arg, "-fallow-so-scripts")) { - graph.allow_so_scripts = true; - } else if (mem.eql(u8, arg, "-fno-allow-so-scripts")) { - graph.allow_so_scripts = false; - } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.startsWith(u8, arg, "-freference-trace=")) { - const num = arg["-freference-trace=".len..]; - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - std.debug.print("unable to parse reference_trace count '{s}': {t}", .{ num, err }); - process.exit(1); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; - } else if (mem.cutPrefix(u8, arg, "-j")) |text| { - const n = std.fmt.parseUnsigned(u32, text, 10) catch |err| - fatal("unable to parse jobs count '{s}': {t}", .{ text, err }); - if (n < 1) fatal("number of jobs must be at least 1", .{}); - threaded.setAsyncLimit(.limited(n)); - graph.max_jobs = n; - } else if (mem.eql(u8, arg, "--")) { - run_args = argsRest(args, arg_idx); - break; - } else { - fatalWithHint("unrecognized argument: '{s}'", .{arg}); - } - } else { - try step_names.append(arena, arg); - } - } - - const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); - const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); - - graph.stderr_mode = switch (color) { - .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), - .on => .escape_codes, - .off => .no_color, - }; - - const scanned_config: ScannedConfig = sc: { - const configuration = c: { - var file = cwd.openFile(io, configure_path, .{}) catch |err| - fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); - defer file.close(io); - break :c Configuration.loadFile(arena, io, file) catch |err| - fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); - }; - var top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index) = .empty; - for (configuration.steps, 0..) |*conf_step, step_index_usize| { - const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); - const flags: Configuration.Step.Flags = @bitCast(configuration.extra[conf_step.extra_index]); - if (flags.tag == .top_level) { - const name = step_index.ptr(&configuration).name.slice(&configuration); - try top_level_steps.put(arena, name, step_index); - } - } - break :sc .{ - .configuration = configuration, - .top_level_steps = top_level_steps, - }; - }; - - if (help_menu) { - var w = initStdoutWriter(io); - scanned_config.printUsage(&graph, w) catch |err| switch (err) { - error.WriteFailed => return stdout_writer_allocation.err.?, - else => |e| return e, - }; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } else if (steps_menu) { - var w = initStdoutWriter(io); - scanned_config.printSteps(&graph, w) catch |err| switch (err) { - error.WriteFailed => return stdout_writer_allocation.err.?, - else => |e| return e, - }; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } else if (print_configuration) { - var w = initStdoutWriter(io); - scanned_config.print(w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return; - } - - if (webui_listen != null) { - if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); - if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); - } - - const main_progress_node = std.Progress.start(io, .{ - .disable_printing = (color == .off), - }); - defer main_progress_node.end(); - - const install_prefix_path: Path = if (graph.environ_map.get("DESTDIR")) |dest_dir| .{ - .root_dir = .cwd(), - .sub_path = try Io.Dir.path.join(arena, &.{ dest_dir, override_install_prefix orelse "/usr" }), - } else if (override_install_prefix) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else .{ - .root_dir = build_root_directory, - .sub_path = "zig-out", - }; - - const install_lib_path: Path = if (override_lib_dir) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else try install_prefix_path.join(arena, "lib"); - - const install_bin_path: Path = if (override_bin_dir) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else try install_prefix_path.join(arena, "bin"); - - const install_include_path: Path = if (override_include_dir) |cwd_relative| .{ - .root_dir = .cwd(), - .sub_path = cwd_relative, - } else try install_prefix_path.join(arena, "include"); - - var maker: Maker = .{ - .gpa = gpa, - .graph = &graph, - .scanned_config = &scanned_config, - .install_paths = .{ - .prefix = install_prefix_path, - .lib = install_lib_path, - .bin = install_bin_path, - .include = install_include_path, - }, - .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), - - .available_rss = max_rss, - .max_rss_is_default = false, - .max_rss_mutex = .init, - .skip_oom_steps = skip_oom_steps, - .unit_test_timeout_ns = test_timeout_ns, - - .watch = watch, - .web_server = undefined, // set after `prepare` - .memory_blocked_steps = .empty, - .step_stack = .empty, - - .error_style = error_style, - .multiline_errors = multiline_errors, - .summary = summary orelse if (watch or webui_listen != null) .line else .failures, - }; - defer { - maker.memory_blocked_steps.deinit(gpa); - maker.step_stack.deinit(gpa); - } - - if (maker.available_rss == 0) { - maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); - maker.max_rss_is_default = true; - } - - maker.prepare(step_names.items) catch |err| switch (err) { - error.DependencyLoopDetected, error.InsufficientMemory => { - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(1); - }, - else => |e| return e, - }; - - var w: Watch = w: { - if (!watch) break :w undefined; - if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(graph.cache.cwd, &scanned_config.configuration, maker.steps); - }; - - const now = Io.Clock.Timestamp.now(io, .awake); - - maker.web_server = if (webui_listen) |listen_address| ws: { - if (builtin.single_threaded) unreachable; // `fatal` above - break :ws .init(.{ - .gpa = gpa, - .graph = &graph, - .all_steps = maker.step_stack.keys(), - .root_prog_node = main_progress_node, - .watch = watch, - .listen_address = listen_address, - .base_timestamp = now, - .configuration = &scanned_config.configuration, - }); - } else null; - - if (maker.web_server) |*ws| { - ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); - } - - rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - try stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H"); - }) { - if (maker.web_server) |*ws| ws.startBuild(); - - try maker.makeStepNames(step_names.items, main_progress_node, fuzz); - - if (maker.web_server) |*web_server| { - if (fuzz) |mode| if (mode != .forever) fatal( - "error: limited fuzzing is not implemented yet for --webui", - .{}, - ); - - web_server.finishBuild(.{ .fuzz = fuzz != null }); - } - - if (maker.web_server) |*ws| { - const c = &scanned_config.configuration; - assert(!watch); // fatal error after CLI parsing - while (true) switch (try ws.wait()) { - .rebuild => { - for (maker.step_stack.keys()) |step_index| { - const step = maker.stepByIndex(step_index); - step.state = .precheck_done; - const deps = step_index.ptr(c).deps.slice(c); - step.pending_deps = @intCast(deps.len); - step.reset(gpa); - } - continue :rebuild; - }, - }; - } - - // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. - if (!Watch.have_impl) unreachable; - - try w.update(gpa, maker.step_stack.keys()); - - // Wait until a file system notification arrives. Read all such events - // until the buffer is empty. Then wait for a debounce interval, resetting - // if any more events come in. After the debounce interval has passed, - // trigger a rebuild on all steps with modified inputs, as well as their - // recursive dependants. - var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; - const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(maker.steps, maker.step_stack.keys()), - }) catch &caption_buf; - var debouncing_node = main_progress_node.start(caption, 0); - var in_debounce = false; - while (true) switch (try w.wait(gpa, io, if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { - .timeout => { - assert(in_debounce); - debouncing_node.end(); - markFailedStepsDirty(gpa, maker.steps, maker.step_stack.keys()); - continue :rebuild; - }, - .dirty => if (!in_debounce) { - in_debounce = true; - debouncing_node.end(); - debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); - }, - .clean => {}, - }; - } -} - -fn markFailedStepsDirty(gpa: Allocator, make_steps: []Step, all_steps: []const Configuration.Step.Index) void { - for (all_steps) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; - switch (step.state) { - .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa), - else => continue, - } - } - // Now that all dirty steps have been found, the remaining steps that - // succeeded from last run shall be marked "cached". - for (all_steps) |step_index| { - const step = &make_steps[@intFromEnum(step_index)]; - switch (step.state) { - .success => step.result_cached = true, - else => continue, - } - } -} - -fn countSubProcesses(make_steps: []Step, all_steps: []const Configuration.Step.Index) usize { - var count: usize = 0; - for (all_steps) |step_index| { - const s = &make_steps[@intFromEnum(step_index)]; - count += @intFromBool(s.getZigProcess() != null); - } - return count; -} - -const InstallPaths = struct { - prefix: Path, - lib: Path, - bin: Path, - include: Path, -}; - -fn stepByIndex(maker: *const Maker, i: Configuration.Step.Index) *Step { - return &maker.steps[@intFromEnum(i)]; -} - -fn prepare(maker: *Maker, step_names: []const []const u8) !void { - const gpa = maker.gpa; - const graph = maker.graph; - const arena = graph.arena; - const seed: u32 = graph.random_seed; - const step_stack = &maker.step_stack; - const c = &maker.scanned_config.configuration; - - @memset(maker.steps, .{}); - - if (step_names.len == 0) { - try step_stack.put(gpa, c.default_step, {}); - } else { - try step_stack.ensureUnusedCapacity(gpa, step_names.len); - for (0..step_names.len) |i| { - const step_name = step_names[step_names.len - i - 1]; - const s = maker.scanned_config.top_level_steps.get(step_name) orelse { - log.info("to list available steps: zig build -l", .{}); - fatal("no such step: {s}", .{step_name}); - }; - step_stack.putAssumeCapacity(s, {}); - } - } - - const starting_steps = try arena.dupe(Configuration.Step.Index, step_stack.keys()); - - var rng = std.Random.DefaultPrng.init(seed); - const rand = rng.random(); - rand.shuffle(Configuration.Step.Index, starting_steps); - - for (starting_steps) |s| { - try constructGraphAndCheckForDependencyLoop(gpa, c, maker.steps, s, &maker.step_stack, rand); - } - - { - // Check that we have enough memory to complete the build. - var any_problems = false; - var max_needed: usize = 0; - for (step_stack.keys()) |step_index| { - const make_step = maker.stepByIndex(step_index); - const conf_step = step_index.ptr(c); - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss == 0) continue; - max_needed = @max(max_needed, max_rss); - if (max_rss > maker.available_rss) { - if (maker.skip_oom_steps) { - make_step.state = .skipped_oom; - for (make_step.dependants.items) |dependant| { - maker.stepByIndex(dependant).pending_deps -= 1; - } - } else { - log.err("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory", .{ - conf_step.owner.depPrefixSlice(c), - conf_step.name.slice(c), - max_rss, - maker.available_rss, - }); - any_problems = true; - } - } - } - if (any_problems) { - if (maker.max_rss_is_default) { - std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ - max_needed, - }); - } - return error.InsufficientMemory; - } - } -} - -fn makeStepNames( - maker: *Maker, - step_names: []const []const u8, - parent_prog_node: std.Progress.Node, - fuzz: ?Fuzz.Mode, -) !void { - const graph = maker.graph; - const gpa = maker.gpa; - const io = graph.io; - const step_stack = &maker.step_stack; - const top_level_steps = &maker.scanned_config.top_level_steps; - const c = &maker.scanned_config.configuration; - - { - // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer, - // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking - // a step is initial when it actually became ready due to an earlier initial step. - var initial_set: std.ArrayList(Configuration.Step.Index) = .empty; - defer initial_set.deinit(gpa); - try initial_set.ensureUnusedCapacity(gpa, step_stack.count()); - for (step_stack.keys()) |step_index| { - const s = maker.stepByIndex(step_index); - if (s.state == .precheck_done and s.pending_deps == 0) { - initial_set.appendAssumeCapacity(step_index); - } - } - - const step_prog = parent_prog_node.start("steps", step_stack.count()); - defer step_prog.end(); - - var group: Io.Group = .init; - defer group.cancel(io); - // Start working on all of the initial steps... - for (initial_set.items) |step_index| try stepReady(maker, &group, step_index, step_prog); - // ...and `makeStep` will trigger every other step when their last dependency finishes. - try group.await(io); - } - - assert(maker.memory_blocked_steps.items.len == 0); - - var test_pass_count: usize = 0; - var test_skip_count: usize = 0; - var test_fail_count: usize = 0; - var test_crash_count: usize = 0; - var test_timeout_count: usize = 0; - - var test_count: usize = 0; - - var success_count: usize = 0; - var skipped_count: usize = 0; - var failure_count: usize = 0; - var pending_count: usize = 0; - var total_compile_errors: usize = 0; - - var cleanup_task = io.async(cleanTmpFiles, .{ io, step_stack.keys() }); - defer cleanup_task.await(io); - - for (step_stack.keys()) |step_index| { - const make_step = maker.stepByIndex(step_index); - test_pass_count += make_step.test_results.passCount(); - test_skip_count += make_step.test_results.skip_count; - test_fail_count += make_step.test_results.fail_count; - test_crash_count += make_step.test_results.crash_count; - test_timeout_count += make_step.test_results.timeout_count; - - test_count += make_step.test_results.test_count; - - switch (make_step.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - .dependency_failure => pending_count += 1, - .success => success_count += 1, - .skipped, .skipped_oom => skipped_count += 1, - .failure => { - failure_count += 1; - const compile_errors_len = make_step.result_error_bundle.errorMessageCount(); - if (compile_errors_len > 0) { - total_compile_errors += compile_errors_len; - } - }, - } - } - - if (fuzz) |mode| blk: { - switch (builtin.os.tag) { - // Current implementation depends on two things that need to be ported to Windows: - // * Memory-mapping to share data between the fuzzer and build runner. - // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving - // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), - else => {}, - } - if (@bitSizeOf(usize) != 64) { - // Current implementation depends on posix.mmap()'s second parameter, `length: usize`, - // being compatible with file system's u64 return value. This is not the case - // on 32-bit platforms. - // Affects or affected by issues #5185, #22523, and #22464. - fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)}); - } - - switch (mode) { - .forever => break :blk, - .limit => {}, - } - - assert(mode == .limit); - var f = Fuzz.init( - gpa, - io, - step_stack.keys(), - parent_prog_node, - mode, - ) catch |err| fatal("failed to start fuzzer: {t}", .{err}); - defer f.deinit(); - - f.start(); - try f.waitAndPrintReport(); - } - - // Every test has a state - assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count); - - if (failure_count == 0) { - std.Progress.setStatus(.success); - } else { - std.Progress.setStatus(.failure); - } - - summary: { - switch (maker.summary) { - .all, .new, .line => {}, - .failures => if (failure_count == 0) break :summary, - .none => break :summary, - } - - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - const t = stderr.terminal(); - const w = &stderr.file_writer.interface; - - const total_count = success_count + failure_count + pending_count + skipped_count; - t.setColor(.cyan) catch {}; - t.setColor(.bold) catch {}; - w.writeAll("Build Summary: ") catch {}; - t.setColor(.reset) catch {}; - w.print("{d}/{d} steps succeeded", .{ success_count, total_count }) catch {}; - { - t.setColor(.dim) catch {}; - var first = true; - if (skipped_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", skipped_count }) catch {}; - first = false; - } - if (failure_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", failure_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - if (test_count > 0) { - w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; - t.setColor(.dim) catch {}; - var first = true; - if (test_skip_count > 0) { - w.print("{s}{d} skipped", .{ if (first) " (" else ", ", test_skip_count }) catch {}; - first = false; - } - if (test_fail_count > 0) { - w.print("{s}{d} failed", .{ if (first) " (" else ", ", test_fail_count }) catch {}; - first = false; - } - if (test_crash_count > 0) { - w.print("{s}{d} crashed", .{ if (first) " (" else ", ", test_crash_count }) catch {}; - first = false; - } - if (test_timeout_count > 0) { - w.print("{s}{d} timed out", .{ if (first) " (" else ", ", test_timeout_count }) catch {}; - first = false; - } - if (!first) w.writeByte(')') catch {}; - t.setColor(.reset) catch {}; - } - - w.writeAll("\n") catch {}; - - if (maker.summary == .line) break :summary; - - // Print a fancy tree with build results. - var step_stack_copy = try step_stack.clone(gpa); - defer step_stack_copy.deinit(gpa); - - var print_node: PrintNode = .{ .parent = null }; - if (step_names.len == 0) { - print_node.last = true; - printTreeStep(maker, c.default_step, t, &print_node, &step_stack_copy) catch |err| switch (err) { - error.Canceled => |e| return e, - else => {}, - }; - } else { - const last_index = if (maker.summary == .all) top_level_steps.count() else blk: { - var i: usize = step_names.len; - while (i > 0) { - i -= 1; - const step_index = top_level_steps.get(step_names[i]).?; - const step = maker.stepByIndex(step_index); - const found = switch (maker.summary) { - .all, .line, .none => unreachable, - .failures => step.state != .success, - .new => !step.result_cached, - }; - if (found) break :blk i; - } - break :blk top_level_steps.count(); - }; - for (step_names, 0..) |step_name, i| { - const step_index = top_level_steps.get(step_name).?; - print_node.last = i + 1 == last_index; - printTreeStep(maker, step_index, t, &print_node, &step_stack_copy) catch |err| switch (err) { - error.Canceled => |e| return e, - else => {}, - }; - } - } - w.writeByte('\n') catch {}; - } - - if (maker.watch or maker.web_server != null) return; - - // Perhaps in the future there could be an Advanced Options flag such as - // --debug-build-runner-leaks which would make this code return instead of - // calling exit. - - const code: u8 = code: { - if (failure_count == 0) break :code 0; // success - if (maker.error_style.verboseContext()) break :code 1; // failure; print build command - break :code 2; // failure; do not print build command - }; - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(code); -} - -fn stepReady( - maker: *Maker, - group: *Io.Group, - step_index: Configuration.Step.Index, - root_prog_node: std.Progress.Node, -) Io.Cancelable!void { - const graph = maker.graph; - const io = graph.io; - const c = &maker.scanned_config.configuration; - const max_rss = step_index.ptr(c).max_rss.toBytes(); - if (max_rss != 0) { - try maker.max_rss_mutex.lock(io); - defer maker.max_rss_mutex.unlock(io); - if (maker.available_rss < max_rss) { - // Running this step right now could possibly exceed the allotted RSS. - maker.memory_blocked_steps.append(maker.gpa, step_index) catch - @panic("TODO eliminate memory allocation here"); - return; - } - maker.available_rss -= max_rss; - } - group.async(io, makeStep, .{ maker, group, step_index, root_prog_node }); -} - -/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready -/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must -/// have already subtracted this value from `maker.available_rss`. This function will release the RSS -/// claim (i.e. add `s.max_rss` back into `maker.available_rss`) and queue any viable memory-blocked -/// steps after "make" completes for `s`. -fn makeStep( - maker: *Maker, - group: *Io.Group, - step_index: Configuration.Step.Index, - root_prog_node: std.Progress.Node, -) Io.Cancelable!void { - const graph = maker.graph; - const io = graph.io; - const gpa = maker.gpa; - const c = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(c); - const step_name = conf_step.name.slice(c); - const deps = conf_step.deps.slice(c); - const make_step = maker.stepByIndex(step_index); - - { - const step_prog_node = root_prog_node.start(step_name, 0); - defer step_prog_node.end(); - - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); - - const new_state: Step.State = for (deps) |dep_index| { - const dep_make_step = maker.stepByIndex(dep_index); - switch (@atomicLoad(Step.State, &dep_make_step.state, .monotonic)) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => break .dependency_failure, - - .success, .skipped => {}, - } - } else if (make_step.make(.{ - .progress_node = step_prog_node, - .watch = maker.watch, - .web_server = if (maker.web_server) |*ws| ws else null, - .unit_test_timeout_ns = maker.unit_test_timeout_ns, - .gpa = gpa, - })) state: { - break :state .success; - } else |err| switch (err) { - error.MakeFailed => .failure, - error.MakeSkipped => .skipped, - }; - - @atomicStore(Step.State, &make_step.state, new_state, .monotonic); - - switch (new_state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .failure, - .dependency_failure, - .skipped_oom, - => { - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); - std.Progress.setStatus(.failure_working); - }, - - .success, - .skipped, - => { - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); - }, - } - } - - // No matter the result, we want to display error/warning messages. - if (make_step.result_error_bundle.errorMessageCount() > 0 or - make_step.result_error_msgs.items.len > 0 or - make_step.result_stderr.len > 0) - { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - printErrorMessages(gpa, c, maker.steps, step_index, .{}, stderr.terminal(), maker.error_style, maker.multiline_errors) catch |err| switch (err) { - error.Canceled => |e| return e, - error.WriteFailed => switch (stderr.file_writer.err.?) { - error.Canceled => |e| return e, - else => {}, - }, - else => {}, - }; - } - - const max_rss = conf_step.max_rss.toBytes(); - if (max_rss != 0) { - var dispatch_set: std.ArrayList(Configuration.Step.Index) = .empty; - defer dispatch_set.deinit(gpa); - - // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set` - // as a staging buffer to avoid recursing into `makeStep` while `maker.max_rss_mutex` is held. - { - try maker.max_rss_mutex.lock(io); - defer maker.max_rss_mutex.unlock(io); - maker.available_rss += max_rss; - dispatch_set.ensureUnusedCapacity(gpa, maker.memory_blocked_steps.items.len) catch - @panic("TODO eliminate memory allocation here"); - while (maker.memory_blocked_steps.getLast()) |candidate_index| { - const candidate_max_rss = candidate_index.ptr(c).max_rss.toBytes(); - if (maker.available_rss < candidate_max_rss) break; - assert(maker.memory_blocked_steps.pop() == candidate_index); - dispatch_set.appendAssumeCapacity(candidate_index); - } - } - for (dispatch_set.items) |candidate| { - group.async(io, makeStep, .{ maker, group, candidate, root_prog_node }); - } - } - - for (make_step.dependants.items) |dependant_index| { - const dependant = maker.stepByIndex(dependant_index); - // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0. - if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) { - try stepReady(maker, group, dependant_index, root_prog_node); - } - } -} - -fn printTreeStep( - maker: *const Maker, - step_index: Configuration.Step.Index, - stderr: Io.Terminal, - parent_node: *PrintNode, - step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), -) !void { - const writer = stderr.writer; - const first = step_stack.swapRemove(step_index); - const summary = maker.summary; - const c = &maker.scanned_config.configuration; - const conf_step = step_index.ptr(c); - const make_step = maker.stepByIndex(step_index); - const skip = switch (summary) { - .none, .line => unreachable, - .all => false, - .new => make_step.result_cached, - .failures => make_step.state == .success, - }; - if (skip) return; - try printPrefix(parent_node, stderr); - - if (parent_node.parent != null) { - if (parent_node.last) { - try printChildNodePrefix(stderr); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x74\x71\x1B\x28\x42 ", // ├─ - else => "+- ", - }); - } - } - - if (!first) try stderr.setColor(.dim); - - // dep_prefix omitted here because it is redundant with the tree. - try writer.writeAll(conf_step.name.slice(c)); - - const deps = conf_step.deps.slice(c); - - if (first) { - try printStepStatus(maker, step_index, stderr); - - const last_index = if (summary == .all) deps.len -| 1 else blk: { - var i: usize = deps.len; - while (i > 0) { - i -= 1; - - const dep_index = deps[i]; - const dep = maker.stepByIndex(dep_index); - const found = switch (summary) { - .all, .line, .none => unreachable, - .failures => dep.state != .success, - .new => !dep.result_cached, - }; - if (found) break :blk i; - } - break :blk deps.len -| 1; - }; - for (deps, 0..) |dep, i| { - var print_node: PrintNode = .{ - .parent = parent_node, - .last = i == last_index, - }; - try printTreeStep(maker, dep, stderr, &print_node, step_stack); - } - } else { - if (deps.len == 0) { - try writer.writeAll(" (reused)\n"); - } else { - try writer.print(" (+{d} more reused dependencies)\n", .{deps.len}); - } - try stderr.setColor(.reset); - } -} - -fn printStepStatus(maker: *const Maker, step_index: Configuration.Step.Index, stderr: Io.Terminal) !void { - const s = maker.stepByIndex(step_index); - const writer = stderr.writer; - switch (s.state) { - .precheck_unstarted => unreachable, - .precheck_started => unreachable, - .precheck_done => unreachable, - - .dependency_failure => { - try stderr.setColor(.dim); - try writer.writeAll(" transitive failure\n"); - try stderr.setColor(.reset); - }, - - .success => { - try stderr.setColor(.green); - if (s.result_cached) { - try writer.writeAll(" cached"); - } else if (s.test_results.test_count > 0) { - const pass_count = s.test_results.passCount(); - assert(s.test_results.test_count == pass_count + s.test_results.skip_count); - try writer.print(" {d} pass", .{pass_count}); - if (s.test_results.skip_count > 0) { - try stderr.setColor(.reset); - try writer.writeAll(", "); - try stderr.setColor(.yellow); - try writer.print("{d} skip", .{s.test_results.skip_count}); - } - try stderr.setColor(.reset); - try writer.print(" ({d} total)", .{s.test_results.test_count}); - } else { - try writer.writeAll(" success"); - } - try stderr.setColor(.reset); - if (s.result_duration_ns) |ns| { - try stderr.setColor(.dim); - if (ns >= std.time.ns_per_min) { - try writer.print(" {d}m", .{ns / std.time.ns_per_min}); - } else if (ns >= std.time.ns_per_s) { - try writer.print(" {d}s", .{ns / std.time.ns_per_s}); - } else if (ns >= std.time.ns_per_ms) { - try writer.print(" {d}ms", .{ns / std.time.ns_per_ms}); - } else if (ns >= std.time.ns_per_us) { - try writer.print(" {d}us", .{ns / std.time.ns_per_us}); - } else { - try writer.print(" {d}ns", .{ns}); - } - try stderr.setColor(.reset); - } - if (s.result_peak_rss != 0) { - const rss = s.result_peak_rss; - try stderr.setColor(.dim); - if (rss >= 1000_000_000) { - try writer.print(" MaxRSS:{d}G", .{rss / 1000_000_000}); - } else if (rss >= 1000_000) { - try writer.print(" MaxRSS:{d}M", .{rss / 1000_000}); - } else if (rss >= 1000) { - try writer.print(" MaxRSS:{d}K", .{rss / 1000}); - } else { - try writer.print(" MaxRSS:{d}B", .{rss}); - } - try stderr.setColor(.reset); - } - try writer.writeAll("\n"); - }, - .skipped => { - try stderr.setColor(.yellow); - try writer.writeAll(" skipped\n"); - try stderr.setColor(.reset); - }, - .skipped_oom => { - const c = &maker.scanned_config.configuration; - const max_rss = step_index.ptr(c).max_rss.toBytes(); - try stderr.setColor(.yellow); - try writer.writeAll(" skipped (not enough memory)"); - try stderr.setColor(.dim); - try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ - max_rss, maker.available_rss, - }); - try stderr.setColor(.reset); - }, - .failure => { - try printStepFailure(maker.steps, step_index, stderr, false); - try stderr.setColor(.reset); - }, - } -} - -fn printStepFailure( - make_steps: []Step, - step_index: Configuration.Step.Index, - stderr: Io.Terminal, - dim: bool, -) !void { - const w = stderr.writer; - const s = &make_steps[@intFromEnum(step_index)]; - if (s.result_error_bundle.errorMessageCount() > 0) { - try stderr.setColor(.red); - try w.print(" {d} errors\n", .{ - s.result_error_bundle.errorMessageCount(), - }); - } else if (!s.test_results.isSuccess()) { - // These first values include all of the test "statuses". Every test is either passsed, - // skipped, failed, crashed, or timed out. - try stderr.setColor(.green); - try w.print(" {d} pass", .{s.test_results.passCount()}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - if (s.test_results.skip_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.yellow); - try w.print("{d} skip", .{s.test_results.skip_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - if (s.test_results.fail_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.red); - try w.print("{d} fail", .{s.test_results.fail_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - if (s.test_results.crash_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.red); - try w.print("{d} crash", .{s.test_results.crash_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - if (s.test_results.timeout_count > 0) { - try w.writeAll(", "); - try stderr.setColor(.red); - try w.print("{d} timeout", .{s.test_results.timeout_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - try w.print(" ({d} total)", .{s.test_results.test_count}); - - // Memory leaks are intentionally written after the total, because is isn't a test *status*, - // but just a flag that any tests -- even passed ones -- can have. We also use a different - // separator, so it looks like: - // 2 pass, 1 skip, 2 fail (5 total); 2 leaks - if (s.test_results.leak_count > 0) { - try w.writeAll("; "); - try stderr.setColor(.red); - try w.print("{d} leaks", .{s.test_results.leak_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - - // It's usually not helpful to know how many error logs there were because they tend to - // just come with other errors (e.g. crashes and leaks print stack traces, and clean - // failures print error traces). So only mention them if they're the only thing causing - // the failure. - const show_err_logs: bool = show: { - var alt_results = s.test_results; - alt_results.log_err_count = 0; - break :show alt_results.isSuccess(); - }; - if (show_err_logs) { - try w.writeAll("; "); - try stderr.setColor(.red); - try w.print("{d} error logs", .{s.test_results.log_err_count}); - try stderr.setColor(.reset); - if (dim) try stderr.setColor(.dim); - } - - try w.writeAll("\n"); - } else if (s.result_error_msgs.items.len > 0) { - try stderr.setColor(.red); - try w.writeAll(" failure\n"); - } else { - assert(s.result_stderr.len > 0); - try stderr.setColor(.red); - try w.writeAll(" w\n"); - } -} - -const PrintNode = struct { - parent: ?*PrintNode, - last: bool = false, -}; - -fn printPrefix(node: *PrintNode, stderr: Io.Terminal) !void { - const parent = node.parent orelse return; - const writer = stderr.writer; - if (parent.parent == null) return; - try printPrefix(parent, stderr); - if (parent.last) { - try writer.writeAll(" "); - } else { - try writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x78\x1B\x28\x42 ", // │ - else => "| ", - }); - } -} - -fn printChildNodePrefix(stderr: Io.Terminal) !void { - try stderr.writer.writeAll(switch (stderr.mode) { - .escape_codes => "\x1B\x28\x30\x6d\x71\x1B\x28\x42 ", // └─ - else => "+- ", - }); -} - -/// Traverse the dependency graph depth-first and make it undirected by having -/// steps know their dependants (they only know dependencies at start). -/// Along the way, check that there is no dependency loop, and record the steps -/// in traversal order in `step_stack`. -/// Each step has its dependencies traversed in random order, this accomplishes -/// two things: -/// - `step_stack` will be in randomized-depth-first order, so the build runner -/// spawns initial steps in a random order -/// - each step's `dependants` list is also filled in a random order, so that -/// when it finishes executing in `makeStep`, it spawns next steps to run in -/// random order -fn constructGraphAndCheckForDependencyLoop( - gpa: Allocator, - c: *const Configuration, - steps: []Step, - step_index: Configuration.Step.Index, - step_stack: *std.AutoArrayHashMapUnmanaged(Configuration.Step.Index, void), - rand: std.Random, -) error{ DependencyLoopDetected, OutOfMemory }!void { - const make_step: *Step = &steps[@intFromEnum(step_index)]; - switch (make_step.state) { - .precheck_started => { - log.err("dependency loop detected: {s}", .{step_index.ptr(c).name.slice(c)}); - return error.DependencyLoopDetected; - }, - .precheck_unstarted => { - make_step.state = .precheck_started; - - const step = step_index.ptr(c); - const dependencies = step.deps.slice(c); - try step_stack.ensureUnusedCapacity(gpa, dependencies.len); - - // We dupe to avoid shuffling the steps in the summary, it depends - // on dependencies' order. - const deps = try gpa.dupe(Configuration.Step.Index, dependencies); - defer gpa.free(deps); - - rand.shuffle(Configuration.Step.Index, deps); - - for (deps) |dep| { - const dep_step: *Step = &steps[@intFromEnum(dep)]; - try step_stack.put(gpa, dep, {}); - try dep_step.dependants.append(gpa, step_index); - constructGraphAndCheckForDependencyLoop(gpa, c, steps, dep, step_stack, rand) catch |err| switch (err) { - error.DependencyLoopDetected => { - log.info("needed by: {s}", .{step_index.ptr(c).name.slice(c)}); - return err; - }, - else => return err, - }; - } - - make_step.state = .precheck_done; - make_step.pending_deps = @intCast(dependencies.len); - }, - .precheck_done => {}, - - // These don't happen until we actually run the step graph. - .dependency_failure => unreachable, - .success => unreachable, - .failure => unreachable, - .skipped => unreachable, - .skipped_oom => unreachable, - } -} - -pub fn printErrorMessages( - gpa: Allocator, - c: *const Configuration, - make_steps: []Step, - failing_step_index: Configuration.Step.Index, - options: std.zig.ErrorBundle.RenderOptions, - stderr: Io.Terminal, - error_style: ErrorStyle, - multiline_errors: MultilineErrors, -) !void { - const writer = stderr.writer; - if (error_style.verboseContext()) { - // Provide context for where these error messages are coming from by - // printing the corresponding Step subtree. - var step_stack: std.ArrayList(Configuration.Step.Index) = .empty; - defer step_stack.deinit(gpa); - try step_stack.append(gpa, failing_step_index); - while (true) { - const last_step = &make_steps[@intFromEnum(step_stack.items[step_stack.items.len - 1])]; - if (last_step.dependants.items.len == 0) break; - try step_stack.append(gpa, last_step.dependants.items[0]); - } - - // Now, `step_stack` has the subtree that we want to print, in reverse order. - try stderr.setColor(.dim); - var indent: usize = 0; - while (step_stack.pop()) |step_index| : (indent += 1) { - if (indent > 0) { - try writer.splatByteAll(' ', (indent - 1) * 3); - try printChildNodePrefix(stderr); - } - - try writer.writeAll(step_index.ptr(c).name.slice(c)); - - if (step_index == failing_step_index) { - try printStepFailure(make_steps, step_index, stderr, true); - } else { - try writer.writeAll("\n"); - } - } - try stderr.setColor(.reset); - } else { - // Just print the failing step itself. - try stderr.setColor(.dim); - try writer.writeAll(failing_step_index.ptr(c).name.slice(c)); - try printStepFailure(make_steps, failing_step_index, stderr, true); - try stderr.setColor(.reset); - } - - const failing_step = &make_steps[@intFromEnum(failing_step_index)]; - - if (failing_step.result_stderr.len > 0) { - try writer.writeAll(failing_step.result_stderr); - if (!mem.endsWith(u8, failing_step.result_stderr, "\n")) { - try writer.writeAll("\n"); - } - } - - try failing_step.result_error_bundle.renderToTerminal(options, stderr); - - for (failing_step.result_error_msgs.items) |msg| { - try stderr.setColor(.red); - try writer.writeAll("error:"); - try stderr.setColor(.reset); - if (std.mem.indexOfScalar(u8, msg, '\n') == null) { - try writer.print(" {s}\n", .{msg}); - } else switch (multiline_errors) { - .indent => { - var it = std.mem.splitScalar(u8, msg, '\n'); - try writer.print(" {s}\n", .{it.first()}); - while (it.next()) |line| { - try writer.print(" {s}\n", .{line}); - } - }, - .newline => try writer.print("\n{s}\n", .{msg}), - .none => try writer.print(" {s}\n", .{msg}), - } - } - - if (error_style.verboseContext()) { - if (failing_step.result_failed_command) |cmd_str| { - try stderr.setColor(.red); - try writer.writeAll("failed command: "); - try stderr.setColor(.reset); - try writer.writeAll(cmd_str); - try writer.writeByte('\n'); - } - } - - try writer.writeByte('\n'); -} - -fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { - if (idx.* >= args.len) return null; - defer idx.* += 1; - return args[idx.*]; -} - -fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); - }; -} - -fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { - const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); - if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); - const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); - return arg; -} - -fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { - if (idx >= args.len) return null; - return args[idx..]; -} - -const Color = std.zig.Color; -const ErrorStyle = enum { - verbose, - minimal, - verbose_clear, - minimal_clear, - fn verboseContext(s: ErrorStyle) bool { - return switch (s) { - .verbose, .verbose_clear => true, - .minimal, .minimal_clear => false, - }; - } - fn clearOnUpdate(s: ErrorStyle) bool { - return switch (s) { - .verbose, .minimal => false, - .verbose_clear, .minimal_clear => true, - }; - } -}; -const MultilineErrors = enum { indent, newline, none }; -const Summary = enum { all, new, failures, line, none }; - -fn fatalWithHint(comptime f: []const u8, args: anytype) noreturn { - log.info("to access the help menu: zig build -h", .{}); - fatal(f, args); -} - -fn cleanTmpFiles(io: Io, steps: []const Configuration.Step.Index) void { - for (steps) |step_index| { - if (true) @panic("TODO"); - const wf = step_index.cast(std.Build.Step.WriteFile) orelse continue; - if (wf.mode != .tmp) continue; - const path = wf.generated_directory.path orelse continue; - Io.Dir.cwd().deleteTree(io, path) catch |err| { - log.warn("failed to delete {s}: {t}", .{ path, err }); - }; - } -} - -var stdio_buffer_allocation: [256]u8 = undefined; -var stdout_writer_allocation: Io.File.Writer = undefined; - -fn initStdoutWriter(io: Io) *Writer { - stdout_writer_allocation = Io.File.stdout().writerStreaming(io, &stdio_buffer_allocation); - return &stdout_writer_allocation.interface; -} - -const ScannedConfig = struct { - configuration: Configuration, - top_level_steps: std.StringArrayHashMapUnmanaged(Configuration.Step.Index), - - fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { - const c = &sc.configuration; - var serializer: std.zon.Serializer = .{ .writer = w }; - var s = try serializer.beginStruct(.{}); - - try s.field("default_step", @intFromEnum(c.default_step), .{}); - { - var ss = try s.beginStructField("top_level_steps", .{}); - for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step| { - try ss.field(name, @intFromEnum(step), .{}); - } - try ss.end(); - } - - try s.end(); - } - - fn printSteps(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - const c = &sc.configuration; - for (sc.top_level_steps.keys(), sc.top_level_steps.values()) |name, step_index| { - const step = step_index.ptr(c); - const decorated_name = if (step_index == c.default_step) - try fmt.allocPrint(arena, "{s} (default)", .{name}) - else - name; - const top_level = c.extraData(Configuration.Step.TopLevel, step.extra_index); - const description = top_level.description.slice(c); - try w.print(" {s:<28} {s}\n", .{ decorated_name, description }); - } - } - - fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { - const arena = graph.arena; - - try w.print( - \\Usage: {s} build [steps] [options] - \\ - \\Steps: - \\ - , .{graph.zig_exe}); - try printSteps(sc, graph, w); - try w.writeAll( - \\ - \\Project-Specific Options: - \\ - ); - - const available_options = sc.configuration.available_options; - if (available_options.len == 0) { - try w.print(" (none)\n", .{}); - } else { - for (available_options) |option| { - const name = option.name.slice(&sc.configuration); - const description = option.description.slice(&sc.configuration); - const help = try fmt.allocPrint(arena, " -D{s}=[{t}]", .{ name, option.type }); - try w.print("{s:<30} {s}\n", .{ help, description }); - if (option.enum_options.slice(&sc.configuration)) |enum_options| { - const padding: [33]u8 = @splat(' '); - try w.writeAll(padding ++ "Supported Values:\n"); - for (enum_options) |enum_option_index| { - const enum_option = enum_option_index.slice(&sc.configuration); - try w.print(padding ++ " {s}\n", .{enum_option}); - } - } - } - } - - try w.writeAll( - \\ - \\System Integration Options: - \\ --search-prefix [path] Add a path to look for binaries, libraries, headers - \\ --sysroot [path] Set the system root directory (usually /) - \\ --libc [file] Provide a file which specifies libc paths - \\ - \\ --system [pkgdir] Disable package fetching; enable all integrations - \\ -fsys=[name] Enable a system integration - \\ -fno-sys=[name] Disable a system integration - \\ - \\ -fdarling, -fno-darling Integration with system-installed Darling to - \\ execute macOS programs on Linux hosts - \\ (default: no) - \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute - \\ foreign-architecture programs on Linux hosts - \\ (default: no) - \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc - \\ (e.g. glibc or musl) built for multiple foreign - \\ architectures, allowing execution of non-native - \\ programs that link with libc. - \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on - \\ ARM64 macOS hosts. (default: no) - \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to - \\ execute WASI binaries. (default: no) - \\ -fwine, -fno-wine Integration with system-installed Wine to execute - \\ Windows programs on Linux hosts. (default: no) - \\ - \\ Available System Integrations: Enabled: - \\ - ); - if (sc.configuration.system_integrations.len == 0) { - try w.writeAll(" (none) -\n"); - } else { - for (sc.configuration.system_integrations) |system_integration| { - const name = system_integration.name.slice(&sc.configuration); - const status = switch (system_integration.status) { - .disabled => "no", - .enabled => "yes", - }; - try w.print(" {s:<43} {s}\n", .{ name, status }); - } - } - - try w.writeAll( - \\ - \\General Options: - \\ -h, --help Print this help to stdout and exit - \\ -l, --list-steps Print available steps to stdout and exit - \\ - \\ -p, --prefix [path] Where to install files (default: zig-out) - \\ --prefix-lib-dir [path] Where to install libraries - \\ --prefix-exe-dir [path] Where to install executables - \\ --prefix-include-dir [path] Where to install C header files - \\ --release[=mode] Request release mode, optionally specifying a - \\ preferred optimization mode: fast, safe, small - \\ - \\ --verbose Print commands before executing them - \\ --color [auto|off|on] Enable or disable colored error messages - \\ --error-style [style] Control how build errors are printed - \\ verbose (Default) Report errors with full context - \\ minimal Report errors after summary, excluding context like command lines - \\ verbose_clear Like 'verbose', but clear the terminal at the start of each update - \\ minimal_clear Like 'minimal', but clear the terminal at the start of each update - \\ --multiline-errors [style] Control how multi-line error messages are printed - \\ indent (Default) Indent non-initial lines to align with initial line - \\ newline Include a leading newline so that the error message is on its own lines - \\ none Print as usual so the first line is misaligned - \\ --summary [mode] Control the printing of the build summary - \\ all Print the build summary in its entirety - \\ new Omit cached steps - \\ failures (Default if short-lived) Only print failed steps - \\ line (Default if long-lived) Only print the single-line summary - \\ none Do not print the build summary - \\ -j<N> Limit concurrent jobs (default is to use all CPU cores) - \\ --maxrss <bytes> Limit memory usage (default is to use available memory) - \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss - \\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded. - \\ The timeout must include a unit: ns, us, ms, s, m, h - \\ --watch Continuously rebuild when source files are modified - \\ --debounce <ms> Delay before rebuilding after changed file detected - \\ --webui[=ip] Enable the web interface on the given IP address - \\ --fuzz[=limit] Continuously search for unit test failures with an optional - \\ limit to the max number of iterations. The argument supports - \\ an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies - \\ '--webui' when no limit is specified. - \\ --time-report Force full rebuild and provide detailed information on - \\ compilation time of Zig source code (implies '--webui') - \\ -fincremental Enable incremental compilation - \\ -fno-incremental Disable incremental compilation - \\ - \\Package Management Options: - \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit - \\ needed (Default) Lazy dependencies are fetched as needed - \\ all Lazy dependencies are always fetched - \\ --fork=[path] Override one or more projects from dependency tree - \\ - \\Advanced Options: - \\ -freference-trace[=num] How many lines of reference trace should be shown per compile error - \\ -fno-reference-trace Disable reference trace - \\ -fallow-so-scripts Allows .so files to be GNU ld scripts - \\ -fno-allow-so-scripts (default) .so files must be ELF files - \\ --build-file [file] Override path to build.zig - \\ --cache-dir [path] Override path to local Zig cache directory - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory - \\ --build-runner [file] Override path to build runner - \\ --seed [integer] For shuffling dependency traversal order (default: random) - \\ --build-id[=style] At a minor link-time expense, embeds a build ID in binaries - \\ fast 8-byte non-cryptographic hash (COFF, ELF, WASM) - \\ sha1, tree 20-byte cryptographic hash (ELF, WASM) - \\ md5 16-byte cryptographic hash (ELF) - \\ uuid 16-byte random UUID (ELF, WASM) - \\ 0x[hexstring] Constant ID, maximum 32 bytes (ELF, WASM) - \\ none (default) No build ID - \\ --debug-log [scope] Enable debugging the compiler - \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --debug-rt Debug compiler runtime libraries - \\ --verbose-link Enable compiler debug output for linking - \\ --verbose-air Enable compiler debug output for Zig AIR - \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR - \\ --verbose-llvm-bc=[file] Enable compiler debug output for LLVM BC - \\ --verbose-cimport Enable compiler debug output for C imports - \\ --verbose-cc Enable compiler debug output for C compilation - \\ --verbose-llvm-cpu-features Enable compiler debug output for LLVM CPU features - \\ - ); - } -}; diff --git a/lib/compiler/maker/Fuzz.zig b/lib/compiler/maker/Fuzz.zig @@ -1,606 +0,0 @@ -const Fuzz = @This(); - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const Build = std.Build; -const Cache = std.Build.Cache; -const Coverage = std.debug.Coverage; -const Configuration = std.Build.Configuration; -const Io = std.Io; -const abi = std.Build.abi.fuzz; -const assert = std.debug.assert; -const fatal = std.process.fatal; -const log = std.log; - -const maker = @import("../maker.zig"); -const WebServer = @import("WebServer.zig"); - -gpa: Allocator, -io: Io, -mode: Mode, - -/// Allocated into `gpa`. -run_steps: []const Configuration.Step.Index, - -group: Io.Group, -root_prog_node: std.Progress.Node, -prog_node: std.Progress.Node, - -/// Protects `coverage_files`. -coverage_mutex: Io.Mutex, -coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap), - -queue_mutex: Io.Mutex, -queue_cond: Io.Condition, -msg_queue: std.ArrayList(Msg), - -pub const Mode = union(enum) { - forever: struct { ws: *WebServer }, - limit: Limited, - - pub const Limited = struct { - amount: u64, - }; -}; - -const Msg = union(enum) { - coverage: struct { - id: u64, - cumulative: struct { - runs: u64, - unique: u64, - coverage: u64, - }, - run: Configuration.Step.Index, - }, - entry_point: struct { - coverage_id: u64, - addr: u64, - }, -}; - -const CoverageMap = struct { - mapped_memory: []align(std.heap.page_size_min) const u8, - coverage: Coverage, - source_locations: []Coverage.SourceLocation, - /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested. - entry_points: std.ArrayList(u32), - start_timestamp: i64, - start_n_runs: u64, - - fn deinit(cm: *CoverageMap, gpa: Allocator) void { - std.posix.munmap(cm.mapped_memory); - cm.coverage.deinit(gpa); - cm.* = undefined; - } -}; - -pub fn init( - gpa: Allocator, - io: Io, - all_steps: []const Configuration.Step.Index, - root_prog_node: std.Progress.Node, - mode: Mode, -) error{ OutOfMemory, Canceled }!Fuzz { - const run_steps: []const Configuration.Step.Index = steps: { - var steps: std.ArrayList(Configuration.Step.Index) = .empty; - defer steps.deinit(gpa); - const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0); - defer rebuild_node.end(); - var rebuild_group: Io.Group = .init; - defer rebuild_group.cancel(io); - - for (all_steps) |step| { - if (true) @panic("TODO"); - const run = step.cast(std.Build.Step.Run) orelse continue; - if (run.producer == null) continue; - if (run.fuzz_tests.items.len == 0) continue; - try steps.append(gpa, run); - rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, rebuild_node }); - } - - if (steps.items.len == 0) fatal("no fuzz tests found", .{}); - rebuild_node.setEstimatedTotalItems(steps.items.len); - const run_steps = try gpa.dupe(Configuration.Step.Index, steps.items); - try rebuild_group.await(io); - break :steps run_steps; - }; - errdefer gpa.free(run_steps); - - for (run_steps) |run_step_index| { - if (true) @panic("TODO"); - assert(run_step_index.fuzz_tests.items.len > 0); - if (run_step_index.rebuilt_executable == null) - fatal("one or more unit tests failed to be rebuilt in fuzz mode", .{}); - } - - return .{ - .gpa = gpa, - .io = io, - .mode = mode, - .run_steps = run_steps, - .group = .init, - .root_prog_node = root_prog_node, - .prog_node = .none, - .coverage_files = .empty, - .coverage_mutex = .init, - .queue_mutex = .init, - .queue_cond = .init, - .msg_queue = .empty, - }; -} - -pub fn start(fuzz: *Fuzz) void { - const io = fuzz.io; - fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", 0); - - if (fuzz.mode == .forever) { - // For polling messages and sending updates to subscribers. - fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err| - fatal("unable to spawn coverage task: {t}", .{err}); - } - - if (true) @panic("TODO"); - - for (fuzz.run_steps) |run| { - assert(run.rebuilt_executable != null); - fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run }); - } -} - -pub fn deinit(fuzz: *Fuzz) void { - const io = fuzz.io; - fuzz.group.cancel(io); - fuzz.prog_node.end(); - fuzz.gpa.free(fuzz.run_steps); -} - -fn rebuildTestsWorkerRun(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) void { - rebuildTestsWorkerRunFallible(run, gpa, parent_prog_node) catch |err| { - const compile = run.producer.?; - log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err }); - }; -} - -fn rebuildTestsWorkerRunFallible(run: Configuration.Step.Index, gpa: Allocator, parent_prog_node: std.Progress.Node) !void { - const graph = run.step.owner.graph; - const io = graph.io; - const compile = run.producer.?; - const prog_node = parent_prog_node.start(compile.step.name, 0); - defer prog_node.end(); - - const result = compile.rebuildInFuzzMode(gpa, prog_node); - - const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; - const show_error_msgs = compile.step.result_error_msgs.items.len > 0; - const show_stderr = compile.step.result_stderr.len > 0; - - if (show_error_msgs or show_compile_errors or show_stderr) { - var buf: [256]u8 = undefined; - const stderr = try io.lockStderr(&buf, graph.stderr_mode); - defer io.unlockStderr(); - maker.printErrorMessages(gpa, &compile.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; - } - - const rebuilt_bin_path = result catch |err| switch (err) { - error.MakeFailed => return, - else => |other| return other, - }; - run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename); -} - -fn fuzzWorkerRun(fuzz: *Fuzz, run: Configuration.Step.Index) void { - const owner = run.step.owner; - const gpa = owner.allocator; - const graph = owner.graph; - const io = graph.io; - - run.rerunInFuzzMode(fuzz, fuzz.prog_node) catch |err| switch (err) { - error.MakeFailed => { - var buf: [256]u8 = undefined; - const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) { - error.Canceled => return, - }; - defer io.unlockStderr(); - maker.printErrorMessages(gpa, &run.step, .{}, stderr.terminal(), .verbose, .indent) catch {}; - return; - }, - else => { - log.err("step '{s}': failed to rerun in fuzz mode: {t}", .{ run.step.name, err }); - return; - }, - }; -} - -pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void { - if (true) @panic("TODO"); - assert(fuzz.mode == .forever); - - var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false); - var dedup_table: DedupTable = .empty; - defer dedup_table.deinit(fuzz.gpa); - - for (fuzz.run_steps) |run_step| { - const compile_inputs = run_step.producer.?.step.inputs.table; - for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| { - try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len); - for (file_list.items) |sub_path| { - if (!std.mem.endsWith(u8, sub_path, ".zig")) continue; - const joined_path = try dir_path.join(arena, sub_path); - dedup_table.putAssumeCapacity(joined_path, {}); - } - } - } - - const deduped_paths = dedup_table.keys(); - const SortContext = struct { - pub fn lessThan(this: @This(), lhs: Build.Cache.Path, rhs: Build.Cache.Path) bool { - _ = this; - return switch (std.mem.order(u8, lhs.root_dir.path orelse ".", rhs.root_dir.path orelse ".")) { - .lt => true, - .gt => false, - .eq => std.mem.lessThan(u8, lhs.sub_path, rhs.sub_path), - }; - } - }; - std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan); - return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths); -} - -pub const Previous = struct { - unique_runs: usize, - entry_points: usize, - sent_source_index: bool, - pub const init: Previous = .{ - .unique_runs = 0, - .entry_points = 0, - .sent_source_index = false, - }; -}; -pub fn sendUpdate( - fuzz: *Fuzz, - socket: *std.http.Server.WebSocket, - prev: *Previous, -) !void { - const io = fuzz.io; - - try fuzz.coverage_mutex.lock(io); - defer fuzz.coverage_mutex.unlock(io); - - const coverage_maps = fuzz.coverage_files.values(); - if (coverage_maps.len == 0) return; - // TODO: handle multiple fuzz steps in the WebSocket packets - const coverage_map = &coverage_maps[0]; - const cov_header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - // TODO: this isn't sound! We need to do volatile reads of these bits rather than handing the - // buffer off to the kernel, because we might race with the fuzzer process[es]. This brings the - // whole mmap strategy into question. Incidentally, I wonder if post-writergate we could pass - // this data straight to the socket with sendfile... - const seen_pcs = cov_header.seenBits(); - const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic); - const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic); - { - if (!prev.sent_source_index) { - prev.sent_source_index = true; - // We need to send initial context. - const header: abi.SourceIndexHeader = .{ - .directories_len = @intCast(coverage_map.coverage.directories.entries.len), - .files_len = @intCast(coverage_map.coverage.files.entries.len), - .source_locations_len = @intCast(coverage_map.source_locations.len), - .string_bytes_len = @intCast(coverage_map.coverage.string_bytes.items.len), - .start_timestamp = coverage_map.start_timestamp, - .start_n_runs = coverage_map.start_n_runs, - }; - var iovecs: [5][]const u8 = .{ - @ptrCast(&header), - @ptrCast(coverage_map.coverage.directories.keys()), - @ptrCast(coverage_map.coverage.files.keys()), - @ptrCast(coverage_map.source_locations), - coverage_map.coverage.string_bytes.items, - }; - try socket.writeMessageVec(&iovecs, .binary); - } - - const header: abi.CoverageUpdateHeader = .{ - .n_runs = n_runs, - .unique_runs = unique_runs, - }; - var iovecs: [2][]const u8 = .{ - @ptrCast(&header), - @ptrCast(seen_pcs), - }; - try socket.writeMessageVec(&iovecs, .binary); - - prev.unique_runs = unique_runs; - } - - if (prev.entry_points != coverage_map.entry_points.items.len) { - const header: abi.EntryPointHeader = .init(@intCast(coverage_map.entry_points.items.len)); - var iovecs: [2][]const u8 = .{ - @ptrCast(&header), - @ptrCast(coverage_map.entry_points.items), - }; - try socket.writeMessageVec(&iovecs, .binary); - - prev.entry_points = coverage_map.entry_points.items.len; - } -} - -fn coverageRun(fuzz: *Fuzz) void { - coverageRunCancelable(fuzz) catch |err| switch (err) { - error.Canceled => return, - }; -} - -fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void { - const io = fuzz.io; - - try fuzz.queue_mutex.lock(io); - defer fuzz.queue_mutex.unlock(io); - - while (true) { - try fuzz.queue_cond.wait(io, &fuzz.queue_mutex); - for (fuzz.msg_queue.items) |msg| switch (msg) { - .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) { - error.AlreadyReported => continue, - error.Canceled => return, - else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), - }, - .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) { - error.AlreadyReported => continue, - error.Canceled => return, - else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}), - }, - }; - fuzz.msg_queue.clearRetainingCapacity(); - } -} -fn prepareTables(fuzz: *Fuzz, run_step_index: Configuration.Step.Index, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void { - if (true) @panic("TODO"); - assert(fuzz.mode == .forever); - const ws = fuzz.mode.forever.ws; - const gpa = fuzz.gpa; - const io = fuzz.io; - - try fuzz.coverage_mutex.lock(io); - defer fuzz.coverage_mutex.unlock(io); - - const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id); - if (gop.found_existing) { - // We are fuzzing the same executable with multiple threads. - // Perhaps the same unit test; perhaps a different one. In any - // case, since the coverage file is the same, we only have to - // notice changes to that one file in order to learn coverage for - // this particular executable. - return; - } - errdefer _ = fuzz.coverage_files.pop(); - - gop.value_ptr.* = .{ - .coverage = std.debug.Coverage.init, - .mapped_memory = undefined, // populated below - .source_locations = undefined, // populated below - .entry_points = .empty, - .start_timestamp = ws.now(), - .start_n_runs = undefined, // populated below - }; - errdefer gop.value_ptr.coverage.deinit(gpa); - - const rebuilt_exe_path = run_step_index.rebuilt_executable.?; - const target = run_step_index.producer.?.rootModuleTarget(); - var debug_info = std.debug.Info.load( - gpa, - io, - rebuilt_exe_path, - &gop.value_ptr.coverage, - target.ofmt, - target.cpu.arch, - ) catch |err| { - log.err("step '{s}': failed to load debug information for '{f}': {t}", .{ - run_step_index.step.name, rebuilt_exe_path, err, - }); - return error.AlreadyReported; - }; - defer debug_info.deinit(gpa); - - const coverage_file_path: Build.Cache.Path = .{ - .root_dir = run_step_index.step.owner.cache_root, - .sub_path = "v/" ++ std.fmt.hex(coverage_id), - }; - var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { - log.err("step '{s}': failed to load coverage file '{f}': {t}", .{ - run_step_index.step.name, coverage_file_path, err, - }); - return error.AlreadyReported; - }; - defer coverage_file.close(io); - - const file_size = coverage_file.length(io) catch |err| { - log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err }); - return error.AlreadyReported; - }; - - const mapped_memory = std.posix.mmap( - null, - file_size, - .{ .READ = true }, - .{ .TYPE = .SHARED }, - coverage_file.handle, - 0, - ) catch |err| { - log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err }); - return error.AlreadyReported; - }; - gop.value_ptr.mapped_memory = mapped_memory; - - const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const pcs = header.pcAddrs(); - const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len); - errdefer gpa.free(source_locations); - - // Unfortunately the PCs array that LLVM gives us from the 8-bit PC - // counters feature is not sorted. - var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .empty; - defer sorted_pcs.deinit(gpa); - try sorted_pcs.resize(gpa, pcs.len); - @memcpy(sorted_pcs.items(.pc), pcs); - for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i); - sorted_pcs.sortUnstable(struct { - addrs: []const u64, - - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return ctx.addrs[a_index] < ctx.addrs[b_index]; - } - }{ .addrs = sorted_pcs.items(.pc) }); - - debug_info.resolveAddresses(gpa, io, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| { - log.err("failed to resolve addresses to source locations: {t}", .{err}); - return error.AlreadyReported; - }; - - for (sorted_pcs.items(.index), sorted_pcs.items(.sl)) |i, sl| source_locations[i] = sl; - gop.value_ptr.source_locations = source_locations; - gop.value_ptr.start_n_runs = header.n_runs; - - ws.notifyUpdate(); -} - -fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void { - const io = fuzz.io; - - try fuzz.coverage_mutex.lock(io); - defer fuzz.coverage_mutex.unlock(io); - - const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?; - const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]); - const pcs = header.pcAddrs(); - - // Since this pcs list is unsorted, we must linear scan for the best index. - const index = i: { - var best: usize = 0; - for (pcs[1..], 1..) |elem_addr, i| { - if (elem_addr == addr) break :i i; - if (elem_addr > addr) continue; - if (elem_addr > pcs[best]) best = i; - } - break :i best; - }; - if (index >= pcs.len) { - log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{ - addr, pcs[0], pcs[pcs.len - 1], - }); - return error.AlreadyReported; - } - if (false) { - const sl = coverage_map.source_locations[index]; - const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename); - if (pcs.len == 1) { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 (final)", .{ - addr, file_name, sl.line, sl.column, - }); - } else if (index == 0) { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index 0 before {x}", .{ - addr, file_name, sl.line, sl.column, pcs[index + 1], - }); - } else if (index == pcs.len - 1) { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} (final) after {x}", .{ - addr, file_name, sl.line, sl.column, index, pcs[index - 1], - }); - } else { - log.debug("server found entry point for 0x{x} at {s}:{d}:{d} - index {d} between {x} and {x}", .{ - addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1], - }); - } - } - try coverage_map.entry_points.append(fuzz.gpa, @intCast(index)); -} - -pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void { - if (true) @panic("TODO"); - assert(fuzz.mode == .limit); - const io = fuzz.io; - - try fuzz.group.await(io); - fuzz.group = .init; - - std.debug.print("======= FUZZING REPORT =======\n", .{}); - for (fuzz.msg_queue.items) |msg| { - if (msg != .coverage) continue; - - const cov = msg.coverage; - const coverage_file_path: std.Build.Cache.Path = .{ - .root_dir = cov.run.step.owner.cache_root, - .sub_path = "v/" ++ std.fmt.hex(cov.id), - }; - var coverage_file = coverage_file_path.root_dir.handle.openFile(io, coverage_file_path.sub_path, .{}) catch |err| { - fatal("step '{s}': failed to load coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, - }); - }; - defer coverage_file.close(io); - - const fuzz_abi = std.Build.abi.fuzz; - var rbuf: [0x1000]u8 = undefined; - var r = coverage_file.reader(io, &rbuf); - - var header: fuzz_abi.SeenPcsHeader = undefined; - r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { - fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, - }); - }; - - if (header.pcs_len == 0) { - fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{ - cov.run.step.name, coverage_file_path, - }); - } - - var seen_count: usize = 0; - const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len); - for (0..chunk_count) |_| { - const seen = r.interface.takeInt(usize, .little) catch |err| { - fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{ - cov.run.step.name, coverage_file_path, err, - }); - }; - seen_count += @popCount(seen); - } - - const seen_f: f64 = @floatFromInt(seen_count); - const total_f: f64 = @floatFromInt(header.pcs_len); - const ratio = seen_f / total_f; - std.debug.print( - \\Step: {s} - \\Fuzz test: "{s}" ({x}) - \\Runs: {} -> {} - \\Unique runs: {} -> {} - \\Coverage: {}/{} -> {}/{} ({:.02}%) - \\ - , .{ - cov.run.step.name, - cov.run.fuzz_tests.items[0], - cov.id, - cov.cumulative.runs, - header.n_runs, - cov.cumulative.unique, - header.unique_runs, - cov.cumulative.coverage, - header.pcs_len, - seen_count, - header.pcs_len, - ratio * 100, - }); - - std.debug.print("------------------------------\n", .{}); - } - std.debug.print( - \\Values are accumulated across multiple runs when preserving the cache. - \\============================== - \\ - , .{}); -} diff --git a/src/main.zig b/src/main.zig @@ -5799,7 +5799,7 @@ fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunn const main_mod_paths: Package.Module.CreateOptions.Paths = .{ .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), - .root_src_path = "maker.zig", + .root_src_path = "Maker.zig", }; const config = try Compilation.Config.resolve(.{