diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c63bff6e0..16c7dd3d0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,12 +84,6 @@ set(ZIG_NO_LIB off CACHE BOOL set(ZIG_NO_LANGREF off CACHE BOOL "Disable copying of langref to the install prefix during the build phase") -set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Deprecated. Use ZIG_NO_LIB") -if(ZIG_SKIP_INSTALL_LIB_FILES) - message(WARNING "ZIG_SKIP_INSTALL_LIB_FILES is deprecated. Use ZIG_NO_LIB instead.") - set(ZIG_NO_LIB ON) -endif() - set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_SHARED_LLVM off CACHE BOOL "Prefer linking against shared LLVM libraries") set(ZIG_STATIC_LLVM ${ZIG_STATIC} CACHE BOOL "Prefer linking against static LLVM libraries") @@ -110,10 +104,6 @@ if (ZIG_SHARED_LLVM AND ZIG_STATIC_LLVM) message(SEND_ERROR "-DZIG_SHARED_LLVM and -DZIG_STATIC_LLVM cannot both be enabled simultaneously") endif() -string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_LIB_DIR_ESCAPED "${ZIG_LIBC_LIB_DIR}") -string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_STATIC_LIB_DIR_ESCAPED "${ZIG_LIBC_STATIC_LIB_DIR}") -string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_INCLUDE_DIR_ESCAPED "${ZIG_LIBC_INCLUDE_DIR}") - set(ZIG_TARGET_TRIPLE "native" CACHE STRING "arch-os-abi to output binaries for") set(ZIG_TARGET_MCPU "native" CACHE STRING "-mcpu parameter to output binaries for") set(ZIG_SINGLE_THREADED off CACHE BOOL "limit the zig compiler to use only 1 thread") @@ -130,6 +120,8 @@ if(ZIG_AR_WORKAROUND) string(REPLACE "" " ar" CMAKE_CXX_ARCHIVE_CREATE ${CMAKE_CXX_ARCHIVE_CREATE}) endif() +set(ZIG_PIE off CACHE BOOL "produce a position independent zig executable") + find_package(llvm 16) find_package(clang 16) find_package(lld 16) @@ -185,11 +177,6 @@ include_directories(${CLANG_INCLUDE_DIRS}) find_package(Threads) set(ZIG_LIB_DIR "lib/zig") -set(C_HEADERS_DEST "${ZIG_LIB_DIR}/include") -set(LIBC_FILES_DEST "${ZIG_LIB_DIR}/libc") -set(LIBUNWIND_FILES_DEST "${ZIG_LIB_DIR}/libunwind") -set(LIBCXX_FILES_DEST "${ZIG_LIB_DIR}/libcxx") -set(ZIG_STD_DEST "${ZIG_LIB_DIR}/std") set(ZIG_CONFIG_H_OUT "${CMAKE_BINARY_DIR}/config.h") set(ZIG_CONFIG_ZIG_OUT "${CMAKE_BINARY_DIR}/config.zig") @@ -686,7 +673,12 @@ if(ZIG_STATIC) endif() add_library(zigcpp STATIC ${ZIG_CPP_SOURCES}) -set_target_properties(zigcpp PROPERTIES COMPILE_FLAGS ${EXE_CXX_FLAGS}) +if(ZIG_PIE) + set(ZIGCPP_CXX_FLAGS "${EXE_CXX_FLAGS} -fPIC") +else() + set(ZIGCPP_CXX_FLAGS "${EXE_CXX_FLAGS}") +endif() +set_target_properties(zigcpp PROPERTIES COMPILE_FLAGS ${ZIGCPP_CXX_FLAGS}) target_link_libraries(zigcpp LINK_PUBLIC ${CLANG_LIBRARIES} @@ -838,10 +830,10 @@ else() set(ZIG_STATIC_ARG "") endif() -if(CMAKE_POSITION_INDEPENDENT_CODE) - set(ZIG_PIE_ARG="-Dpie") +if(CMAKE_POSITION_INDEPENDENT_CODE OR ZIG_PIE) + set(ZIG_PIE_ARG "-Dpie") else() - set(ZIG_PIE_ARG="") + set(ZIG_PIE_ARG "") endif() set(ZIG_BUILD_ARGS @@ -860,8 +852,12 @@ set(ZIG_BUILD_ARGS ) add_custom_target(stage3 ALL - COMMAND zig2 build compile ${ZIG_BUILD_ARGS} - DEPENDS zig2 + DEPENDS "${CMAKE_BINARY_DIR}/stage3/bin/zig" +) + +add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/stage3/bin/zig" + COMMAND zig2 build --prefix "${CMAKE_BINARY_DIR}/stage3" ${ZIG_BUILD_ARGS} COMMENT STATUS "Building stage3" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) diff --git a/build.zig b/build.zig index 6cb3e35488..28bc528772 100644 --- a/build.zig +++ b/build.zig @@ -9,7 +9,7 @@ const fs = std.fs; const InstallDirectoryOptions = std.Build.InstallDirectoryOptions; const assert = std.debug.assert; -const zig_version = std.builtin.Version{ .major = 0, .minor = 11, .patch = 0 }; +const zig_version = std.SemanticVersion{ .major = 0, .minor = 11, .patch = 0 }; const stack_size = 32 * 1024 * 1024; pub fn build(b: *std.Build) !void { @@ -28,12 +28,9 @@ pub fn build(b: *std.Build) !void { const use_zig_libcxx = b.option(bool, "use-zig-libcxx", "If libc++ is needed, use zig's bundled version, don't try to integrate with the system") orelse false; const test_step = b.step("test", "Run all the tests"); - const deprecated_skip_install_lib_files = b.option(bool, "skip-install-lib-files", "deprecated. see no-lib") orelse false; - if (deprecated_skip_install_lib_files) { - std.log.warn("-Dskip-install-lib-files is deprecated in favor of -Dno-lib", .{}); - } - const skip_install_lib_files = b.option(bool, "no-lib", "skip copying of lib/ files and langref to installation prefix. Useful for development") orelse deprecated_skip_install_lib_files; + const skip_install_lib_files = b.option(bool, "no-lib", "skip copying of lib/ files and langref to installation prefix. Useful for development") orelse false; const skip_install_langref = b.option(bool, "no-langref", "skip copying of langref to the installation prefix") orelse skip_install_lib_files; + const no_bin = b.option(bool, "no-bin", "skip emitting compiler binary") orelse false; const docgen_exe = b.addExecutable(.{ .name = "docgen", @@ -58,12 +55,6 @@ pub fn build(b: *std.Build) !void { const docs_step = b.step("docs", "Build documentation"); docs_step.dependOn(&docgen_cmd.step); - // This is for legacy reasons, to be removed after our CI scripts are upgraded to use - // the file from the install prefix instead. - const legacy_write_to_cache = b.addWriteFiles(); - legacy_write_to_cache.addCopyFileToSource(langref_file, "zig-cache/langref.html"); - docs_step.dependOn(&legacy_write_to_cache.step); - const check_case_exe = b.addExecutable(.{ .name = "check-case", .root_source_file = .{ .path = "test/src/Cases.zig" }, @@ -175,12 +166,16 @@ pub fn build(b: *std.Build) !void { exe.strip = strip; exe.pie = pie; exe.sanitize_thread = sanitize_thread; - exe.build_id = b.option(bool, "build-id", "Include a build id note") orelse false; exe.entitlements = entitlements; - b.installArtifact(exe); + if (no_bin) exe.emit_bin = .no_emit; - const compile_step = b.step("compile", "Build the self-hosted compiler"); - compile_step.dependOn(&exe.step); + exe.build_id = b.option( + std.Build.Step.Compile.BuildId, + "build-id", + "Request creation of '.note.gnu.build-id' section", + ); + + b.installArtifact(exe); test_step.dependOn(&exe.step); @@ -204,7 +199,7 @@ pub fn build(b: *std.Build) !void { exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa); exe_options.addOption(bool, "force_gpa", force_gpa); exe_options.addOption(bool, "only_c", only_c); - exe_options.addOption(bool, "omit_pkg_fetching_code", only_c); + exe_options.addOption(bool, "only_core_functionality", only_c); if (link_libc) { exe.linkLibC(); @@ -242,12 +237,12 @@ pub fn build(b: *std.Build) !void { }, 2 => { // Untagged development build (e.g. 0.10.0-dev.2025+ecf0050a9). - var it = mem.split(u8, git_describe, "-"); + var it = mem.splitScalar(u8, git_describe, '-'); const tagged_ancestor = it.first(); const commit_height = it.next().?; const commit_id = it.next().?; - const ancestor_ver = try std.builtin.Version.parse(tagged_ancestor); + const ancestor_ver = try std.SemanticVersion.parse(tagged_ancestor); if (zig_version.order(ancestor_ver) != .gt) { std.debug.print("Zig version '{}' must be greater than tagged ancestor '{}'\n", .{ zig_version, ancestor_ver }); std.process.exit(1); @@ -287,7 +282,7 @@ pub fn build(b: *std.Build) !void { // That means we also have to rely on stage1 compiled c++ files. We parse config.h to find // the information passed on to us from cmake. if (cfg.cmake_prefix_path.len > 0) { - var it = mem.tokenize(u8, cfg.cmake_prefix_path, ";"); + var it = mem.tokenizeScalar(u8, cfg.cmake_prefix_path, ';'); while (it.next()) |path| { b.addSearchPrefix(path); } @@ -360,7 +355,7 @@ pub fn build(b: *std.Build) !void { test_cases_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa); test_cases_options.addOption(bool, "force_gpa", force_gpa); test_cases_options.addOption(bool, "only_c", only_c); - test_cases_options.addOption(bool, "omit_pkg_fetching_code", true); + test_cases_options.addOption(bool, "only_core_functionality", true); test_cases_options.addOption(bool, "enable_qemu", b.enable_qemu); test_cases_options.addOption(bool, "enable_wine", b.enable_wine); test_cases_options.addOption(bool, "enable_wasmtime", b.enable_wasmtime); @@ -475,9 +470,8 @@ pub fn build(b: *std.Build) !void { .skip_non_native = skip_non_native, .skip_cross_glibc = skip_cross_glibc, .skip_libc = skip_libc, - // I observed a value of 3398275072 on my M1, and multiplied by 1.1 to - // get this amount: - .max_rss = 3738102579, + // I observed a value of 3932766208 on the M1 CI. + .max_rss = 4080218931, })); try addWasiUpdateStep(b, version); @@ -512,7 +506,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void { exe_options.addOption(bool, "enable_tracy_callstack", false); exe_options.addOption(bool, "enable_tracy_allocation", false); exe_options.addOption(bool, "value_tracing", false); - exe_options.addOption(bool, "omit_pkg_fetching_code", true); + exe_options.addOption(bool, "only_core_functionality", true); const run_opt = b.addSystemCommand(&.{ "wasm-opt", @@ -536,7 +530,7 @@ fn addCompilerStep( b: *std.Build, optimize: std.builtin.OptimizeMode, target: std.zig.CrossTarget, -) *std.Build.CompileStep { +) *std.Build.Step.Compile { const exe = b.addExecutable(.{ .name = "zig", .root_source_file = .{ .path = "src/main.zig" }, @@ -564,7 +558,7 @@ const exe_cflags = [_][]const u8{ fn addCmakeCfgOptionsToExe( b: *std.Build, cfg: CMakeConfig, - exe: *std.Build.CompileStep, + exe: *std.Build.Step.Compile, use_zig_libcxx: bool, ) !void { if (exe.target.isDarwin()) { @@ -643,7 +637,7 @@ fn addCmakeCfgOptionsToExe( } } -fn addStaticLlvmOptionsToExe(exe: *std.Build.CompileStep) !void { +fn addStaticLlvmOptionsToExe(exe: *std.Build.Step.Compile) !void { // Adds the Zig C++ sources which both stage1 and stage2 need. // // We need this because otherwise zig_clang_cc1_main.cpp ends up pulling @@ -682,7 +676,7 @@ fn addStaticLlvmOptionsToExe(exe: *std.Build.CompileStep) !void { fn addCxxKnownPath( b: *std.Build, ctx: CMakeConfig, - exe: *std.Build.CompileStep, + exe: *std.Build.Step.Compile, objname: []const u8, errtxt: ?[]const u8, need_cpp_includes: bool, @@ -690,7 +684,7 @@ fn addCxxKnownPath( if (!std.process.can_spawn) return error.RequiredLibraryNotFound; const path_padded = b.exec(&.{ ctx.cxx_compiler, b.fmt("-print-file-name={s}", .{objname}) }); - var tokenizer = mem.tokenize(u8, path_padded, "\r\n"); + var tokenizer = mem.tokenizeAny(u8, path_padded, "\r\n"); const path_unpadded = tokenizer.next().?; if (mem.eql(u8, path_unpadded, objname)) { if (errtxt) |msg| { @@ -712,8 +706,8 @@ fn addCxxKnownPath( } } -fn addCMakeLibraryList(exe: *std.Build.CompileStep, list: []const u8) void { - var it = mem.tokenize(u8, list, ";"); +fn addCMakeLibraryList(exe: *std.Build.Step.Compile, list: []const u8) void { + var it = mem.tokenizeScalar(u8, list, ';'); while (it.next()) |lib| { if (mem.startsWith(u8, lib, "-l")) { exe.linkSystemLibrary(lib["-l".len..]); @@ -726,7 +720,7 @@ fn addCMakeLibraryList(exe: *std.Build.CompileStep, list: []const u8) void { } const CMakeConfig = struct { - llvm_linkage: std.Build.CompileStep.Linkage, + llvm_linkage: std.Build.Step.Compile.Linkage, cmake_binary_dir: []const u8, cmake_prefix_path: []const u8, cmake_static_library_prefix: []const u8, @@ -858,18 +852,18 @@ fn parseConfigH(b: *std.Build, config_h_text: []const u8) ?CMakeConfig { // .prefix = ZIG_LLVM_LINK_MODE parsed manually below }; - var lines_it = mem.tokenize(u8, config_h_text, "\r\n"); + var lines_it = mem.tokenizeAny(u8, config_h_text, "\r\n"); while (lines_it.next()) |line| { inline for (mappings) |mapping| { if (mem.startsWith(u8, line, mapping.prefix)) { - var it = mem.split(u8, line, "\""); + var it = mem.splitScalar(u8, line, '"'); _ = it.first(); // skip the stuff before the quote const quoted = it.next().?; // the stuff inside the quote @field(ctx, mapping.field) = toNativePathSep(b, quoted); } } if (mem.startsWith(u8, line, "#define ZIG_LLVM_LINK_MODE ")) { - var it = mem.split(u8, line, "\""); + var it = mem.splitScalar(u8, line, '"'); _ = it.next().?; // skip the stuff before the quote const quoted = it.next().?; // the stuff inside the quote ctx.llvm_linkage = if (mem.eql(u8, quoted, "shared")) .dynamic else .static; diff --git a/doc/docgen.zig b/doc/docgen.zig index 07636fd152..93cc316de1 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -111,13 +111,13 @@ const Token = struct { end: usize, const Id = enum { - Invalid, - Content, - BracketOpen, - TagContent, - Separator, - BracketClose, - Eof, + invalid, + content, + bracket_open, + tag_content, + separator, + bracket_close, + eof, }; }; @@ -129,18 +129,18 @@ const Tokenizer = struct { code_node_count: usize, const State = enum { - Start, - LBracket, - Hash, - TagName, - Eof, + start, + l_bracket, + hash, + tag_name, + eof, }; fn init(source_file_name: []const u8, buffer: []const u8) Tokenizer { return Tokenizer{ .buffer = buffer, .index = 0, - .state = State.Start, + .state = .start, .source_file_name = source_file_name, .code_node_count = 0, }; @@ -148,84 +148,84 @@ const Tokenizer = struct { fn next(self: *Tokenizer) Token { var result = Token{ - .id = Token.Id.Eof, + .id = .eof, .start = self.index, .end = undefined, }; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (self.state) { - State.Start => switch (c) { + .start => switch (c) { '{' => { - self.state = State.LBracket; + self.state = .l_bracket; }, else => { - result.id = Token.Id.Content; + result.id = .content; }, }, - State.LBracket => switch (c) { + .l_bracket => switch (c) { '#' => { - if (result.id != Token.Id.Eof) { + if (result.id != .eof) { self.index -= 1; - self.state = State.Start; + self.state = .start; break; } else { - result.id = Token.Id.BracketOpen; + result.id = .bracket_open; self.index += 1; - self.state = State.TagName; + self.state = .tag_name; break; } }, else => { - result.id = Token.Id.Content; - self.state = State.Start; + result.id = .content; + self.state = .start; }, }, - State.TagName => switch (c) { + .tag_name => switch (c) { '|' => { - if (result.id != Token.Id.Eof) { + if (result.id != .eof) { break; } else { - result.id = Token.Id.Separator; + result.id = .separator; self.index += 1; break; } }, '#' => { - self.state = State.Hash; + self.state = .hash; }, else => { - result.id = Token.Id.TagContent; + result.id = .tag_content; }, }, - State.Hash => switch (c) { + .hash => switch (c) { '}' => { - if (result.id != Token.Id.Eof) { + if (result.id != .eof) { self.index -= 1; - self.state = State.TagName; + self.state = .tag_name; break; } else { - result.id = Token.Id.BracketClose; + result.id = .bracket_close; self.index += 1; - self.state = State.Start; + self.state = .start; break; } }, else => { - result.id = Token.Id.TagContent; - self.state = State.TagName; + result.id = .tag_content; + self.state = .tag_name; }, }, - State.Eof => unreachable, + .eof => unreachable, } } else { switch (self.state) { - State.Start, State.LBracket, State.Eof => {}, + .start, .l_bracket, .eof => {}, else => { - result.id = Token.Id.Invalid; + result.id = .invalid; }, } - self.state = State.Eof; + self.state = .eof; } result.end = self.index; return result; @@ -276,7 +276,7 @@ fn parseError(tokenizer: *Tokenizer, token: Token, comptime fmt: []const u8, arg } } { - const caret_count = std.math.min(token.end, loc.line_end) - token.start; + const caret_count = @min(token.end, loc.line_end) - token.start; var i: usize = 0; while (i < caret_count) : (i += 1) { print("~", .{}); @@ -311,9 +311,9 @@ const SeeAlsoItem = struct { }; const ExpectedOutcome = enum { - Succeed, - Fail, - BuildFail, + succeed, + fail, + build_fail, }; const Code = struct { @@ -331,12 +331,12 @@ const Code = struct { additional_options: []const []const u8, const Id = union(enum) { - Test, - TestError: []const u8, - TestSafety: []const u8, - Exe: ExpectedOutcome, - Obj: ?[]const u8, - Lib, + @"test", + test_error: []const u8, + test_safety: []const u8, + exe: ExpectedOutcome, + obj: ?[]const u8, + lib, }; }; @@ -379,8 +379,8 @@ const Toc = struct { }; const Action = enum { - Open, - Close, + open, + close, }; fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { @@ -388,7 +388,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { errdefer urls.deinit(); var header_stack_size: usize = 0; - var last_action = Action.Open; + var last_action: Action = .open; var last_columns: ?u8 = null; var toc_buf = std.ArrayList(u8).init(allocator); @@ -404,38 +404,38 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { while (true) { const token = tokenizer.next(); switch (token.id) { - Token.Id.Eof => { + .eof => { if (header_stack_size != 0) { return parseError(tokenizer, token, "unbalanced headers", .{}); } try toc.writeAll(" \n"); break; }, - Token.Id.Content => { + .content => { try nodes.append(Node{ .Content = tokenizer.buffer[token.start..token.end] }); }, - Token.Id.BracketOpen => { - const tag_token = try eatToken(tokenizer, Token.Id.TagContent); + .bracket_open => { + const tag_token = try eatToken(tokenizer, .tag_content); const tag_name = tokenizer.buffer[tag_token.start..tag_token.end]; if (mem.eql(u8, tag_name, "nav")) { - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); try nodes.append(Node.Nav); } else if (mem.eql(u8, tag_name, "builtin")) { - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); try nodes.append(Node{ .Builtin = tag_token }); } else if (mem.eql(u8, tag_name, "header_open")) { - _ = try eatToken(tokenizer, Token.Id.Separator); - const content_token = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .separator); + const content_token = try eatToken(tokenizer, .tag_content); const content = tokenizer.buffer[content_token.start..content_token.end]; var columns: ?u8 = null; while (true) { const bracket_tok = tokenizer.next(); switch (bracket_tok.id) { - .BracketClose => break, - .Separator => continue, - .TagContent => { + .bracket_close => break, + .separator => continue, + .tag_content => { const param = tokenizer.buffer[bracket_tok.start..bracket_tok.end]; if (mem.eql(u8, param, "2col")) { columns = 2; @@ -467,7 +467,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { parseError(tokenizer, kv.value, "other tag here", .{}) catch {}; return error.ParseError; } - if (last_action == Action.Open) { + if (last_action == .open) { try toc.writeByte('\n'); try toc.writeByteNTimes(' ', header_stack_size * 4); if (last_columns) |n| { @@ -476,7 +476,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { try toc.writeAll("\n"); } else { try toc.writeAll("\n"); - last_action = Action.Close; + last_action = .close; } } else if (mem.eql(u8, tag_name, "see_also")) { var list = std.ArrayList(SeeAlsoItem).init(allocator); @@ -502,15 +502,15 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { while (true) { const see_also_tok = tokenizer.next(); switch (see_also_tok.id) { - Token.Id.TagContent => { + .tag_content => { const content = tokenizer.buffer[see_also_tok.start..see_also_tok.end]; try list.append(SeeAlsoItem{ .name = content, .token = see_also_tok, }); }, - Token.Id.Separator => {}, - Token.Id.BracketClose => { + .separator => {}, + .bracket_close => { try nodes.append(Node{ .SeeAlso = try list.toOwnedSlice() }); break; }, @@ -518,17 +518,17 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { } } } else if (mem.eql(u8, tag_name, "link")) { - _ = try eatToken(tokenizer, Token.Id.Separator); - const name_tok = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .separator); + const name_tok = try eatToken(tokenizer, .tag_content); const name = tokenizer.buffer[name_tok.start..name_tok.end]; const url_name = blk: { const tok = tokenizer.next(); switch (tok.id) { - Token.Id.BracketClose => break :blk name, - Token.Id.Separator => { - const explicit_text = try eatToken(tokenizer, Token.Id.TagContent); - _ = try eatToken(tokenizer, Token.Id.BracketClose); + .bracket_close => break :blk name, + .separator => { + const explicit_text = try eatToken(tokenizer, .tag_content); + _ = try eatToken(tokenizer, .bracket_close); break :blk tokenizer.buffer[explicit_text.start..explicit_text.end]; }, else => return parseError(tokenizer, tok, "invalid link token", .{}), @@ -543,45 +543,45 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { }, }); } else if (mem.eql(u8, tag_name, "code_begin")) { - _ = try eatToken(tokenizer, Token.Id.Separator); - const code_kind_tok = try eatToken(tokenizer, Token.Id.TagContent); - _ = try eatToken(tokenizer, Token.Id.Separator); - const name_tok = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .separator); + const code_kind_tok = try eatToken(tokenizer, .tag_content); + _ = try eatToken(tokenizer, .separator); + const name_tok = try eatToken(tokenizer, .tag_content); const name = tokenizer.buffer[name_tok.start..name_tok.end]; var error_str: []const u8 = ""; const maybe_sep = tokenizer.next(); switch (maybe_sep.id) { - Token.Id.Separator => { - const error_tok = try eatToken(tokenizer, Token.Id.TagContent); + .separator => { + const error_tok = try eatToken(tokenizer, .tag_content); error_str = tokenizer.buffer[error_tok.start..error_tok.end]; - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); }, - Token.Id.BracketClose => {}, + .bracket_close => {}, else => return parseError(tokenizer, token, "invalid token", .{}), } const code_kind_str = tokenizer.buffer[code_kind_tok.start..code_kind_tok.end]; var code_kind_id: Code.Id = undefined; var just_check_syntax = false; if (mem.eql(u8, code_kind_str, "exe")) { - code_kind_id = Code.Id{ .Exe = ExpectedOutcome.Succeed }; + code_kind_id = Code.Id{ .exe = .succeed }; } else if (mem.eql(u8, code_kind_str, "exe_err")) { - code_kind_id = Code.Id{ .Exe = ExpectedOutcome.Fail }; + code_kind_id = Code.Id{ .exe = .fail }; } else if (mem.eql(u8, code_kind_str, "exe_build_err")) { - code_kind_id = Code.Id{ .Exe = ExpectedOutcome.BuildFail }; + code_kind_id = Code.Id{ .exe = .build_fail }; } else if (mem.eql(u8, code_kind_str, "test")) { - code_kind_id = Code.Id.Test; + code_kind_id = .@"test"; } else if (mem.eql(u8, code_kind_str, "test_err")) { - code_kind_id = Code.Id{ .TestError = error_str }; + code_kind_id = Code.Id{ .test_error = error_str }; } else if (mem.eql(u8, code_kind_str, "test_safety")) { - code_kind_id = Code.Id{ .TestSafety = error_str }; + code_kind_id = Code.Id{ .test_safety = error_str }; } else if (mem.eql(u8, code_kind_str, "obj")) { - code_kind_id = Code.Id{ .Obj = null }; + code_kind_id = Code.Id{ .obj = null }; } else if (mem.eql(u8, code_kind_str, "obj_err")) { - code_kind_id = Code.Id{ .Obj = error_str }; + code_kind_id = Code.Id{ .obj = error_str }; } else if (mem.eql(u8, code_kind_str, "lib")) { - code_kind_id = Code.Id.Lib; + code_kind_id = Code.Id.lib; } else if (mem.eql(u8, code_kind_str, "syntax")) { - code_kind_id = Code.Id{ .Obj = null }; + code_kind_id = Code.Id{ .obj = null }; just_check_syntax = true; } else { return parseError(tokenizer, code_kind_tok, "unrecognized code kind: {s}", .{code_kind_str}); @@ -599,9 +599,9 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { defer additional_options.deinit(); const source_token = while (true) { - const content_tok = try eatToken(tokenizer, Token.Id.Content); - _ = try eatToken(tokenizer, Token.Id.BracketOpen); - const end_code_tag = try eatToken(tokenizer, Token.Id.TagContent); + const content_tok = try eatToken(tokenizer, .content); + _ = try eatToken(tokenizer, .bracket_open); + const end_code_tag = try eatToken(tokenizer, .tag_content); const end_tag_name = tokenizer.buffer[end_code_tag.start..end_code_tag.end]; if (mem.eql(u8, end_tag_name, "code_release_fast")) { mode = .ReleaseFast; @@ -612,8 +612,8 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { } else if (mem.eql(u8, end_tag_name, "code_verbose_cimport")) { verbose_cimport = true; } else if (mem.eql(u8, end_tag_name, "code_link_object")) { - _ = try eatToken(tokenizer, Token.Id.Separator); - const obj_tok = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .separator); + const obj_tok = try eatToken(tokenizer, .tag_content); try link_objects.append(tokenizer.buffer[obj_tok.start..obj_tok.end]); } else if (mem.eql(u8, end_tag_name, "target_windows")) { target_str = "x86_64-windows"; @@ -630,11 +630,11 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { } else if (mem.eql(u8, end_tag_name, "link_mode_dynamic")) { link_mode = .Dynamic; } else if (mem.eql(u8, end_tag_name, "additonal_option")) { - _ = try eatToken(tokenizer, Token.Id.Separator); - const option = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .separator); + const option = try eatToken(tokenizer, .tag_content); try additional_options.append(tokenizer.buffer[option.start..option.end]); } else if (mem.eql(u8, end_tag_name, "code_end")) { - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); break content_tok; } else { return parseError( @@ -644,7 +644,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { .{end_tag_name}, ); } - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); } else unreachable; // TODO issue #707 try nodes.append(Node{ .Code = Code{ @@ -664,10 +664,10 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { }); tokenizer.code_node_count += 1; } else if (mem.eql(u8, tag_name, "syntax")) { - _ = try eatToken(tokenizer, Token.Id.BracketClose); - const content_tok = try eatToken(tokenizer, Token.Id.Content); - _ = try eatToken(tokenizer, Token.Id.BracketOpen); - const end_syntax_tag = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .bracket_close); + const content_tok = try eatToken(tokenizer, .content); + _ = try eatToken(tokenizer, .bracket_open); + const end_syntax_tag = try eatToken(tokenizer, .tag_content); const end_tag_name = tokenizer.buffer[end_syntax_tag.start..end_syntax_tag.end]; if (!mem.eql(u8, end_tag_name, "endsyntax")) { return parseError( @@ -677,13 +677,13 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { .{end_tag_name}, ); } - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); try nodes.append(Node{ .InlineSyntax = content_tok }); } else if (mem.eql(u8, tag_name, "shell_samp")) { - _ = try eatToken(tokenizer, Token.Id.BracketClose); - const content_tok = try eatToken(tokenizer, Token.Id.Content); - _ = try eatToken(tokenizer, Token.Id.BracketOpen); - const end_syntax_tag = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .bracket_close); + const content_tok = try eatToken(tokenizer, .content); + _ = try eatToken(tokenizer, .bracket_open); + const end_syntax_tag = try eatToken(tokenizer, .tag_content); const end_tag_name = tokenizer.buffer[end_syntax_tag.start..end_syntax_tag.end]; if (!mem.eql(u8, end_tag_name, "end_shell_samp")) { return parseError( @@ -693,20 +693,20 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { .{end_tag_name}, ); } - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); try nodes.append(Node{ .Shell = content_tok }); } else if (mem.eql(u8, tag_name, "syntax_block")) { - _ = try eatToken(tokenizer, Token.Id.Separator); - const source_type_tok = try eatToken(tokenizer, Token.Id.TagContent); + _ = try eatToken(tokenizer, .separator); + const source_type_tok = try eatToken(tokenizer, .tag_content); var name: []const u8 = "sample_code"; const maybe_sep = tokenizer.next(); switch (maybe_sep.id) { - Token.Id.Separator => { - const name_tok = try eatToken(tokenizer, Token.Id.TagContent); + .separator => { + const name_tok = try eatToken(tokenizer, .tag_content); name = tokenizer.buffer[name_tok.start..name_tok.end]; - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); }, - Token.Id.BracketClose => {}, + .bracket_close => {}, else => return parseError(tokenizer, token, "invalid token", .{}), } const source_type_str = tokenizer.buffer[source_type_tok.start..source_type_tok.end]; @@ -723,12 +723,12 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { return parseError(tokenizer, source_type_tok, "unrecognized code kind: {s}", .{source_type_str}); } const source_token = while (true) { - const content_tok = try eatToken(tokenizer, Token.Id.Content); - _ = try eatToken(tokenizer, Token.Id.BracketOpen); - const end_code_tag = try eatToken(tokenizer, Token.Id.TagContent); + const content_tok = try eatToken(tokenizer, .content); + _ = try eatToken(tokenizer, .bracket_open); + const end_code_tag = try eatToken(tokenizer, .tag_content); const end_tag_name = tokenizer.buffer[end_code_tag.start..end_code_tag.end]; if (mem.eql(u8, end_tag_name, "end_syntax_block")) { - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); break content_tok; } else { return parseError( @@ -738,7 +738,7 @@ fn genToc(allocator: Allocator, tokenizer: *Tokenizer) !Toc { .{end_tag_name}, ); } - _ = try eatToken(tokenizer, Token.Id.BracketClose); + _ = try eatToken(tokenizer, .bracket_close); }; try nodes.append(Node{ .SyntaxBlock = SyntaxBlock{ .source_type = source_type, .name = name, .source_token = source_token } }); } else { @@ -796,28 +796,37 @@ fn writeEscaped(out: anytype, input: []const u8) !void { } } -//#define VT_RED "\x1b[31;1m" -//#define VT_GREEN "\x1b[32;1m" -//#define VT_CYAN "\x1b[36;1m" -//#define VT_WHITE "\x1b[37;1m" -//#define VT_BOLD "\x1b[0;1m" -//#define VT_RESET "\x1b[0m" - -test "term color" { - const input_bytes = "A\x1b[32;1mgreen\x1b[0mB"; - const result = try termColor(std.testing.allocator, input_bytes); - defer std.testing.allocator.free(result); - try testing.expectEqualSlices(u8, "AgreenB", result); +// Returns true if number is in slice. +fn in(slice: []const u8, number: u8) bool { + for (slice) |n| { + if (number == n) return true; + } + return false; } fn termColor(allocator: Allocator, input: []const u8) ![]u8 { + // The SRG sequences generates by the Zig compiler are in the format: + // ESC [ ; m + // or + // ESC [ m + // + // where + // foreground-color is 31 (red), 32 (green), 36 (cyan) + // n is 0 (reset), 1 (bold), 2 (dim) + // + // Note that 37 (white) is currently not used by the compiler. + // + // See std.debug.TTY.Color. + const supported_sgr_colors = [_]u8{ 31, 32, 36 }; + const supported_sgr_numbers = [_]u8{ 0, 1, 2 }; + var buf = std.ArrayList(u8).init(allocator); defer buf.deinit(); var out = buf.writer(); - var number_start_index: usize = undefined; - var first_number: usize = undefined; - var second_number: usize = undefined; + var sgr_param_start_index: usize = undefined; + var sgr_num: u8 = undefined; + var sgr_color: u8 = undefined; var i: usize = 0; var state: enum { start, @@ -848,7 +857,7 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 { }, .lbracket => switch (c) { '0'...'9' => { - number_start_index = i; + sgr_param_start_index = i; state = .number; }, else => return error.UnsupportedEscape, @@ -856,13 +865,12 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 { .number => switch (c) { '0'...'9' => {}, else => { - first_number = std.fmt.parseInt(usize, input[number_start_index..i], 10) catch unreachable; - second_number = 0; + sgr_num = try std.fmt.parseInt(u8, input[sgr_param_start_index..i], 10); + sgr_color = 0; state = .after_number; i -= 1; }, }, - .after_number => switch (c) { ';' => state = .arg, 'D' => state = .start, @@ -877,7 +885,7 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 { }, .arg => switch (c) { '0'...'9' => { - number_start_index = i; + sgr_param_start_index = i; state = .arg_number; }, else => return error.UnsupportedEscape, @@ -885,7 +893,15 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 { .arg_number => switch (c) { '0'...'9' => {}, else => { - second_number = std.fmt.parseInt(usize, input[number_start_index..i], 10) catch unreachable; + // Keep the sequence consistent, foreground color first. + // 32;1m is equivalent to 1;32m, but the latter will + // generate an incorrect HTML class without notice. + sgr_color = sgr_num; + if (!in(&supported_sgr_colors, sgr_color)) return error.UnsupportedForegroundColor; + + sgr_num = try std.fmt.parseInt(u8, input[sgr_param_start_index..i], 10); + if (!in(&supported_sgr_numbers, sgr_num)) return error.UnsupportedNumber; + state = .expect_end; i -= 1; }, @@ -896,10 +912,16 @@ fn termColor(allocator: Allocator, input: []const u8) ![]u8 { while (open_span_count != 0) : (open_span_count -= 1) { try out.writeAll(""); } - if (first_number != 0 or second_number != 0) { - try out.print("", .{ first_number, second_number }); - open_span_count += 1; + if (sgr_num == 0) { + if (sgr_color != 0) return error.UnsupportedColor; + continue; } + if (sgr_color != 0) { + try out.print("", .{ sgr_color, sgr_num }); + } else { + try out.print("", .{sgr_num}); + } + open_span_count += 1; }, else => return error.UnsupportedEscape, }, @@ -1223,45 +1245,43 @@ fn printShell(out: anytype, shell_content: []const u8, escape: bool) !void { const trimmed_shell_content = mem.trim(u8, shell_content, " \n"); try out.writeAll("
Shell
");
     var cmd_cont: bool = false;
-    var iter = std.mem.split(u8, trimmed_shell_content, "\n");
+    var iter = std.mem.splitScalar(u8, trimmed_shell_content, '\n');
     while (iter.next()) |orig_line| {
         const line = mem.trimRight(u8, orig_line, " ");
         if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] != '\\') {
-            try out.writeAll(start_line ++ "$ ");
+            try out.writeAll("$ ");
             const s = std.mem.trimLeft(u8, line[1..], " ");
             if (escape) {
                 try writeEscaped(out, s);
             } else {
                 try out.writeAll(s);
             }
-            try out.writeAll("" ++ end_line ++ "\n");
+            try out.writeAll("" ++ "\n");
         } else if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] == '\\') {
-            try out.writeAll(start_line ++ "$ ");
+            try out.writeAll("$ ");
             const s = std.mem.trimLeft(u8, line[1..], " ");
             if (escape) {
                 try writeEscaped(out, s);
             } else {
                 try out.writeAll(s);
             }
-            try out.writeAll(end_line ++ "\n");
+            try out.writeAll("\n");
             cmd_cont = true;
         } else if (line.len > 0 and line[line.len - 1] != '\\' and cmd_cont) {
-            try out.writeAll(start_line);
             if (escape) {
                 try writeEscaped(out, line);
             } else {
                 try out.writeAll(line);
             }
-            try out.writeAll("" ++ end_line ++ "\n");
+            try out.writeAll("" ++ "\n");
             cmd_cont = false;
         } else {
-            try out.writeAll(start_line);
             if (escape) {
                 try writeEscaped(out, line);
             } else {
                 try out.writeAll(line);
             }
-            try out.writeAll(end_line ++ "\n");
+            try out.writeAll("\n");
         }
     }
 
@@ -1371,7 +1391,7 @@ fn genHtml(
                 var shell_out = shell_buffer.writer();
 
                 switch (code.id) {
-                    Code.Id.Exe => |expected_outcome| code_block: {
+                    .exe => |expected_outcome| code_block: {
                         var build_args = std.ArrayList([]const u8).init(allocator);
                         defer build_args.deinit();
                         try build_args.appendSlice(&[_][]const u8{
@@ -1420,7 +1440,7 @@ fn genHtml(
 
                         try shell_out.print("\n", .{});
 
-                        if (expected_outcome == .BuildFail) {
+                        if (expected_outcome == .build_fail) {
                             const result = try ChildProcess.exec(.{
                                 .allocator = allocator,
                                 .argv = build_args.items,
@@ -1476,7 +1496,7 @@ fn genHtml(
 
                         var exited_with_signal = false;
 
-                        const result = if (expected_outcome == ExpectedOutcome.Fail) blk: {
+                        const result = if (expected_outcome == .fail) blk: {
                             const result = try ChildProcess.exec(.{
                                 .allocator = allocator,
                                 .argv = run_args,
@@ -1507,13 +1527,13 @@ fn genHtml(
                         const colored_stderr = try termColor(allocator, escaped_stderr);
                         const colored_stdout = try termColor(allocator, escaped_stdout);
 
-                        try shell_out.print("\n$ ./{s}\n{s}{s}", .{ code.name, colored_stdout, colored_stderr });
+                        try shell_out.print("$ ./{s}\n{s}{s}", .{ code.name, colored_stdout, colored_stderr });
                         if (exited_with_signal) {
                             try shell_out.print("(process terminated by signal)", .{});
                         }
                         try shell_out.writeAll("\n");
                     },
-                    Code.Id.Test => {
+                    .@"test" => {
                         var test_args = std.ArrayList([]const u8).init(allocator);
                         defer test_args.deinit();
 
@@ -1565,7 +1585,7 @@ fn genHtml(
                         const escaped_stdout = try escapeHtml(allocator, result.stdout);
                         try shell_out.print("\n{s}{s}\n", .{ escaped_stderr, escaped_stdout });
                     },
-                    Code.Id.TestError => |error_match| {
+                    .test_error => |error_match| {
                         var test_args = std.ArrayList([]const u8).init(allocator);
                         defer test_args.deinit();
 
@@ -1621,8 +1641,7 @@ fn genHtml(
                         const colored_stderr = try termColor(allocator, escaped_stderr);
                         try shell_out.print("\n{s}\n", .{colored_stderr});
                     },
-
-                    Code.Id.TestSafety => |error_match| {
+                    .test_safety => |error_match| {
                         var test_args = std.ArrayList([]const u8).init(allocator);
                         defer test_args.deinit();
 
@@ -1685,7 +1704,7 @@ fn genHtml(
                             colored_stderr,
                         });
                     },
-                    Code.Id.Obj => |maybe_error_match| {
+                    .obj => |maybe_error_match| {
                         const name_plus_obj_ext = try std.fmt.allocPrint(allocator, "{s}{s}", .{ code.name, obj_ext });
                         var build_args = std.ArrayList([]const u8).init(allocator);
                         defer build_args.deinit();
@@ -1758,7 +1777,7 @@ fn genHtml(
                         }
                         try shell_out.writeAll("\n");
                     },
-                    Code.Id.Lib => {
+                    .lib => {
                         const bin_basename = try std.zig.binNameAlloc(allocator, .{
                             .root_name = code.name,
                             .target = builtin.target,
@@ -1878,7 +1897,193 @@ fn dumpArgs(args: []const []const u8) void {
         print("\n", .{});
 }
 
-test "shell parsed" {
+test "term supported colors" {
+    const test_allocator = testing.allocator;
+
+    {
+        const input = "A\x1b[31;1mred\x1b[0mB";
+        const expect = "AredB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[32;1mgreen\x1b[0mB";
+        const expect = "AgreenB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[36;1mcyan\x1b[0mB";
+        const expect = "AcyanB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[1mbold\x1b[0mB";
+        const expect = "AboldB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        const input = "A\x1b[2mdim\x1b[0mB";
+        const expect = "AdimB";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+}
+
+test "term output from zig" {
+    // Use data generated by https://github.com/perillo/zig-tty-test-data,
+    // with zig version 0.11.0-dev.1898+36d47dd19.
+    const test_allocator = testing.allocator;
+
+    {
+        // 1.1-with-build-progress.out
+        const input = "Semantic Analysis [1324] \x1b[25D\x1b[0KLLVM Emit Object... \x1b[20D\x1b[0KLLVM Emit Object... \x1b[20D\x1b[0KLLD Link... \x1b[12D\x1b[0K";
+        const expect = "";
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 2.1-with-reference-traces.out
+        const input = "\x1b[1msrc/2.1-with-reference-traces.zig:3:7: \x1b[31;1merror: \x1b[0m\x1b[1mcannot assign to constant\n\x1b[0m    x += 1;\n    \x1b[32;1m~~^~~~\n\x1b[0m\x1b[0m\x1b[2mreferenced by:\n    main: src/2.1-with-reference-traces.zig:7:5\n    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17\n    remaining reference traces hidden; use '-freference-trace' to see all reference traces\n\n\x1b[0m";
+        const expect =
+            \\src/2.1-with-reference-traces.zig:3:7: error: cannot assign to constant
+            \\    x += 1;
+            \\    ~~^~~~
+            \\referenced by:
+            \\    main: src/2.1-with-reference-traces.zig:7:5
+            \\    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17
+            \\    remaining reference traces hidden; use '-freference-trace' to see all reference traces
+            \\
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 2.2-without-reference-traces.out
+        const input = "\x1b[1m/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:128:29: \x1b[31;1merror: \x1b[0m\x1b[1minvalid type given to fixedBufferStream\n\x1b[0m                    else => @compileError(\"invalid type given to fixedBufferStream\"),\n                            \x1b[32;1m^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\x1b[0m\x1b[1m/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:116:66: \x1b[36;1mnote: \x1b[0m\x1b[1mcalled from here\n\x1b[0mpub fn fixedBufferStream(buffer: anytype) FixedBufferStream(Slice(@TypeOf(buffer))) {\n;                                                            \x1b[32;1m~~~~~^~~~~~~~~~~~~~~~~\n\x1b[0m";
+        const expect =
+            \\/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:128:29: error: invalid type given to fixedBufferStream
+            \\                    else => @compileError("invalid type given to fixedBufferStream"),
+            \\                            ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+            \\/usr/local/lib/zig/lib/std/io/fixed_buffer_stream.zig:116:66: note: called from here
+            \\pub fn fixedBufferStream(buffer: anytype) FixedBufferStream(Slice(@TypeOf(buffer))) {
+            \\;                                                            ~~~~~^~~~~~~~~~~~~~~~~
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 2.3-with-notes.out
+        const input = "\x1b[1msrc/2.3-with-notes.zig:6:9: \x1b[31;1merror: \x1b[0m\x1b[1mexpected type '*2.3-with-notes.Derp', found '*2.3-with-notes.Wat'\n\x1b[0m    bar(w);\n        \x1b[32;1m^\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:6:9: \x1b[36;1mnote: \x1b[0m\x1b[1mpointer type child '2.3-with-notes.Wat' cannot cast into pointer type child '2.3-with-notes.Derp'\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:2:13: \x1b[36;1mnote: \x1b[0m\x1b[1mopaque declared here\n\x1b[0mconst Wat = opaque {};\n            \x1b[32;1m^~~~~~~~~\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:1:14: \x1b[36;1mnote: \x1b[0m\x1b[1mopaque declared here\n\x1b[0mconst Derp = opaque {};\n             \x1b[32;1m^~~~~~~~~\n\x1b[0m\x1b[1msrc/2.3-with-notes.zig:4:18: \x1b[36;1mnote: \x1b[0m\x1b[1mparameter type declared here\n\x1b[0mextern fn bar(d: *Derp) void;\n                 \x1b[32;1m^~~~~\n\x1b[0m\x1b[0m\x1b[2mreferenced by:\n    main: src/2.3-with-notes.zig:10:5\n    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17\n    remaining reference traces hidden; use '-freference-trace' to see all reference traces\n\n\x1b[0m";
+        const expect =
+            \\src/2.3-with-notes.zig:6:9: error: expected type '*2.3-with-notes.Derp', found '*2.3-with-notes.Wat'
+            \\    bar(w);
+            \\        ^
+            \\src/2.3-with-notes.zig:6:9: note: pointer type child '2.3-with-notes.Wat' cannot cast into pointer type child '2.3-with-notes.Derp'
+            \\src/2.3-with-notes.zig:2:13: note: opaque declared here
+            \\const Wat = opaque {};
+            \\            ^~~~~~~~~
+            \\src/2.3-with-notes.zig:1:14: note: opaque declared here
+            \\const Derp = opaque {};
+            \\             ^~~~~~~~~
+            \\src/2.3-with-notes.zig:4:18: note: parameter type declared here
+            \\extern fn bar(d: *Derp) void;
+            \\                 ^~~~~
+            \\referenced by:
+            \\    main: src/2.3-with-notes.zig:10:5
+            \\    callMain: /usr/local/lib/zig/lib/std/start.zig:607:17
+            \\    remaining reference traces hidden; use '-freference-trace' to see all reference traces
+            \\
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 3.1-with-error-return-traces.out
+
+        const input = "error: Error\n\x1b[1m/home/zig/src/3.1-with-error-return-traces.zig:5:5\x1b[0m: \x1b[2m0x20b008 in callee (3.1-with-error-return-traces)\x1b[0m\n    return error.Error;\n    \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.1-with-error-return-traces.zig:9:5\x1b[0m: \x1b[2m0x20b113 in caller (3.1-with-error-return-traces)\x1b[0m\n    try callee();\n    \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.1-with-error-return-traces.zig:13:5\x1b[0m: \x1b[2m0x20b153 in main (3.1-with-error-return-traces)\x1b[0m\n    try caller();\n    \x1b[32;1m^\x1b[0m\n";
+        const expect =
+            \\error: Error
+            \\/home/zig/src/3.1-with-error-return-traces.zig:5:5: 0x20b008 in callee (3.1-with-error-return-traces)
+            \\    return error.Error;
+            \\    ^
+            \\/home/zig/src/3.1-with-error-return-traces.zig:9:5: 0x20b113 in caller (3.1-with-error-return-traces)
+            \\    try callee();
+            \\    ^
+            \\/home/zig/src/3.1-with-error-return-traces.zig:13:5: 0x20b153 in main (3.1-with-error-return-traces)
+            \\    try caller();
+            \\    ^
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+
+    {
+        // 3.2-with-stack-trace.out
+        const input = "\x1b[1m/usr/local/lib/zig/lib/std/debug.zig:561:19\x1b[0m: \x1b[2m0x22a107 in writeCurrentStackTrace__anon_5898 (3.2-with-stack-trace)\x1b[0m\n    while (it.next()) |return_address| {\n                  \x1b[32;1m^\x1b[0m\n\x1b[1m/usr/local/lib/zig/lib/std/debug.zig:157:80\x1b[0m: \x1b[2m0x20bb23 in dumpCurrentStackTrace (3.2-with-stack-trace)\x1b[0m\n        writeCurrentStackTrace(stderr, debug_info, detectTTYConfig(io.getStdErr()), start_addr) catch |err| {\n                                                                               \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.2-with-stack-trace.zig:5:36\x1b[0m: \x1b[2m0x20d3b2 in foo (3.2-with-stack-trace)\x1b[0m\n    std.debug.dumpCurrentStackTrace(null);\n                                   \x1b[32;1m^\x1b[0m\n\x1b[1m/home/zig/src/3.2-with-stack-trace.zig:9:8\x1b[0m: \x1b[2m0x20b458 in main (3.2-with-stack-trace)\x1b[0m\n    foo();\n       \x1b[32;1m^\x1b[0m\n\x1b[1m/usr/local/lib/zig/lib/std/start.zig:607:22\x1b[0m: \x1b[2m0x20a965 in posixCallMainAndExit (3.2-with-stack-trace)\x1b[0m\n            root.main();\n                     \x1b[32;1m^\x1b[0m\n\x1b[1m/usr/local/lib/zig/lib/std/start.zig:376:5\x1b[0m: \x1b[2m0x20a411 in _start (3.2-with-stack-trace)\x1b[0m\n    @call(.never_inline, posixCallMainAndExit, .{});\n    \x1b[32;1m^\x1b[0m\n";
+        const expect =
+            \\/usr/local/lib/zig/lib/std/debug.zig:561:19: 0x22a107 in writeCurrentStackTrace__anon_5898 (3.2-with-stack-trace)
+            \\    while (it.next()) |return_address| {
+            \\                  ^
+            \\/usr/local/lib/zig/lib/std/debug.zig:157:80: 0x20bb23 in dumpCurrentStackTrace (3.2-with-stack-trace)
+            \\        writeCurrentStackTrace(stderr, debug_info, detectTTYConfig(io.getStdErr()), start_addr) catch |err| {
+            \\                                                                               ^
+            \\/home/zig/src/3.2-with-stack-trace.zig:5:36: 0x20d3b2 in foo (3.2-with-stack-trace)
+            \\    std.debug.dumpCurrentStackTrace(null);
+            \\                                   ^
+            \\/home/zig/src/3.2-with-stack-trace.zig:9:8: 0x20b458 in main (3.2-with-stack-trace)
+            \\    foo();
+            \\       ^
+            \\/usr/local/lib/zig/lib/std/start.zig:607:22: 0x20a965 in posixCallMainAndExit (3.2-with-stack-trace)
+            \\            root.main();
+            \\                     ^
+            \\/usr/local/lib/zig/lib/std/start.zig:376:5: 0x20a411 in _start (3.2-with-stack-trace)
+            \\    @call(.never_inline, posixCallMainAndExit, .{});
+            \\    ^
+            \\
+        ;
+
+        const result = try termColor(test_allocator, input);
+        defer test_allocator.free(result);
+        try testing.expectEqualSlices(u8, expect, result);
+    }
+}
+
+test "printShell" {
     const test_allocator = std.testing.allocator;
 
     {
@@ -1886,7 +2091,7 @@ test "shell parsed" {
             \\$ zig build test.zig
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
+            \\
Shell
$ zig build test.zig
             \\

         ;
 
@@ -1894,7 +2099,6 @@ test "shell parsed" {
         defer buffer.deinit();
 
         try printShell(buffer.writer(), shell_out, false);
-        std.log.emerg("{s}", .{buffer.items});
         try testing.expectEqualSlices(u8, expected, buffer.items);
     }
     {
@@ -1903,8 +2107,8 @@ test "shell parsed" {
             \\build output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\build output
+            \\
Shell
$ zig build test.zig
+            \\build output
             \\

         ;
 
@@ -1921,9 +2125,9 @@ test "shell parsed" {
             \\$ ./test
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\build output
-            \\$ ./test
+            \\
Shell
$ zig build test.zig
+            \\build output
+            \\$ ./test
             \\

         ;
 
@@ -1941,10 +2145,10 @@ test "shell parsed" {
             \\output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\
-            \\$ ./test
-            \\output
+            \\
Shell
$ zig build test.zig
+            \\
+            \\$ ./test
+            \\output
             \\

         ;
 
@@ -1961,9 +2165,9 @@ test "shell parsed" {
             \\output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\$ ./test
-            \\output
+            \\
Shell
$ zig build test.zig
+            \\$ ./test
+            \\output
             \\

         ;
 
@@ -1982,11 +2186,11 @@ test "shell parsed" {
             \\output
         ;
         const expected =
-            \\
Shell
$ zig build test.zig \
-            \\ --build-option
-            \\build output
-            \\$ ./test
-            \\output
+            \\
Shell
$ zig build test.zig \
+            \\ --build-option
+            \\build output
+            \\$ ./test
+            \\output
             \\

         ;
 
@@ -2000,15 +2204,15 @@ test "shell parsed" {
         // intentional space after "--build-option1 \"
         const shell_out =
             \\$ zig build test.zig \
-            \\ --build-option1 \
+            \\ --build-option1 \ 
             \\ --build-option2
             \\$ ./test
         ;
         const expected =
-            \\
Shell
$ zig build test.zig \
-            \\ --build-option1 \
-            \\ --build-option2
-            \\$ ./test
+            \\
Shell
$ zig build test.zig \
+            \\ --build-option1 \
+            \\ --build-option2
+            \\$ ./test
             \\

         ;
 
@@ -2024,8 +2228,8 @@ test "shell parsed" {
             \\$ ./test
         ;
         const expected =
-            \\
Shell
$ zig build test.zig \
-            \\$ ./test
+            \\
Shell
$ zig build test.zig \
+            \\$ ./test
             \\

         ;
 
@@ -2042,9 +2246,9 @@ test "shell parsed" {
             \\$1
         ;
         const expected =
-            \\
Shell
$ zig build test.zig
-            \\$ ./test
-            \\$1
+            \\
Shell
$ zig build test.zig
+            \\$ ./test
+            \\$1
             \\

         ;
 
@@ -2059,7 +2263,7 @@ test "shell parsed" {
             \\$zig build test.zig
         ;
         const expected =
-            \\
Shell
$zig build test.zig
+            \\
Shell
$zig build test.zig
             \\

         ;
 
diff --git a/doc/langref.html.in b/doc/langref.html.in
index a213950a15..bd66603f51 100644
--- a/doc/langref.html.in
+++ b/doc/langref.html.in
@@ -71,19 +71,19 @@
           text-align: left;
           font-weight: normal;
       }
-      .t0_1, .t37, .t37_1 {
+      .sgr-1m {
         font-weight: bold;
       }
-      .t2_0 {
+      .sgr-2m {
         color: #575757;
       }
-      .t31_1 {
+      .sgr-31_1m {
         color: #b40000;
       }
-      .t32_1 {
+      .sgr-32_1m {
         color: green;
       }
-      .t36_1 {
+      .sgr-36_1m {
         color: #005C7A;
       }
       .file {
@@ -114,7 +114,7 @@
         line-height: normal;
       }
       kbd {
-        font-weight: bold;
+        font-weight: normal;
       }
       .table-wrapper {
         width: 100%;
@@ -232,16 +232,16 @@
         table, th, td {
             border-color: grey;
         }
-        .t2_0 {
+        .sgr-2m {
             color: grey;
         }
-        .t31_1 {
+        .sgr-31_1m {
             color: red;
         }
-        .t32_1 {
+        .sgr-32_1m {
             color: #00B800;
         }
-        .t36_1 {
+        .sgr-36_1m {
             color: #0086b3;
         }
         code {
@@ -413,7 +413,7 @@ pub fn main() !void {
       

         The code sample shows the contents of a file named hello.zig. Files storing Zig
         source code are {#link|UTF-8 encoded|Source Encoding#} text files. The files storing
-        Zig source code are usually named with the .zig extension.
+        Zig source code must be named with the .zig extension.
       

       

         Following the hello.zig Zig code sample, the {#link|Zig Build System#} is used
@@ -487,7 +487,7 @@ pub fn main() !void {
         purposely written to show how to perform {#link|string|String Literals and Unicode Code Point Literals#}
         substitution in the {#syntax#}print{#endsyntax#} function. The curly-braces inside of the first argument
         are substituted with the compile-time known value inside of the second argument
-        (known as an {#link|tuple|Tuples#}). The \n
+        (known as a {#link|tuple|Tuples#}). The \n
         inside of the double-quotes of the first argument is the {#link|escape sequence|Escape Sequences#} for the
         newline character. The {#link|try#} expression evaluates the result of {#syntax#}stdout.print{#endsyntax#}.
         If the result is an error, then the {#syntax#}try{#endsyntax#} expression will return from
@@ -518,6 +518,14 @@ pub fn main() void {
       {#see_also|Values|@import|Errors|Root Source File|Source Encoding#}
       {#header_close#}
       {#header_open|Comments#}
+      

+      Zig supports 3 types of comments. Normal comments are ignored, but doc comments
+      and top-level doc comments are used by the compiler to generate the package documentation.
+      

+      

+      The generated documentation is still experimental, and can be produced with:
+      

+      {#shell_samp#}zig test -femit-docs main.zig{#end_shell_samp#}
       {#code_begin|exe|comments#}
 const print = @import("std").debug.print;
 
@@ -535,7 +543,7 @@ pub fn main() void {
       comments in C).  This helps allow Zig to have the property that each line
       of code can be tokenized out of context.
       

-      {#header_open|Doc comments#}
+      {#header_open|Doc Comments#}
       

       A doc comment is one that begins with exactly three slashes (i.e.
       {#syntax#}///{#endsyntax#} but not {#syntax#}////{#endsyntax#});
@@ -562,21 +570,44 @@ const Timestamp = struct {
 };
       {#code_end#}
       

-      Doc comments are only allowed in certain places; eventually, it will
-      become a compile error to have a doc comment in an unexpected place, such as
-      in the middle of an expression, or just before a non-doc comment.
+      Doc comments are only allowed in certain places; it is a compile error to
+      have a doc comment in an unexpected place, such as in the middle of an expression,
+      or just before a non-doc comment.
+      

+      {#code_begin|obj_err|invalid_doc-comment|expected type expression, found 'a document comment'#}
+/// doc-comment
+//! top-level doc-comment
+const std = @import("std");
+      {#code_end#}
+      {#code_begin|obj_err|unattached_doc-comment|unattached documentation comment#}
+pub fn main() void {}
+
+/// End of file
+      {#code_end#}
+      

+      Doc comments can be interleaved with normal comments. Currently, when producing
+      the package documentation, normal comments are merged with doc comments.
       

       {#header_close#}
       {#header_open|Top-Level Doc Comments#}
-      
User documentation that doesn't belong to whatever
-      immediately follows it, like {#link|container|Containers#}-level documentation, goes
-      in top-level doc comments. A top-level doc comment is one that
-      begins with two slashes and an exclamation point:
-      {#syntax#}//!{#endsyntax#}.

+      

+      A top-level doc comment is one that begins with two slashes and an exclamation
+      point: {#syntax#}//!{#endsyntax#}; it documents the current module.
+      

+      

+      It is a compile error if a top-level doc comment is not placed at the start
+      of a {#link|container|Containers#}, before any expressions.
+      

       {#code_begin|syntax|tldoc_comments#}
 //! This module provides functions for retrieving the current date and
 //! time with varying degrees of precision and accuracy. It does not
 //! depend on libc, but will use functions from it if available.
+
+const S = struct {
+    //! Top level comments are allowed inside a container other than a module,
+    //! but it is not very useful.  Currently, when producing the package
+    //! documentation, these comments are ignored.
+};
       {#code_end#}
       {#header_close#}
       {#header_close#}
@@ -1060,13 +1091,18 @@ test "expect addOne adds one to 41" {
     try std.testing.expect(addOne(41) == 42);
 }
 
+test addOne {
+    // A test name can also be written using an identifier.
+    try std.testing.expect(addOne(41) == 42);
+}
+
 /// The function `addOne` adds one to the number given as its argument.
 fn addOne(number: i32) i32 {
     return number + 1;
 }
       {#code_end#}
       

-        The introducing_zig_test.zig code sample tests the {#link|function|Functions#}
+        The testing_introduction.zig code sample tests the {#link|function|Functions#}
         {#syntax#}addOne{#endsyntax#} to ensure that it returns {#syntax#}42{#endsyntax#} given the input
         {#syntax#}41{#endsyntax#}. From this test's perspective, the {#syntax#}addOne{#endsyntax#} function is
         said to be code under test.
@@ -1087,20 +1123,25 @@ fn addOne(number: i32) i32 {
         printed to standard error by the default test runner:
       

       

-        
Test [1/1] test "expect addOne adds one to 41"...

+        
Test [1/2] test.expect addOne adds one to 41...

         
Lines like this indicate which test, out of the total number of tests, is being run.
-          In this case, [1/1] indicates that the first test, out of a total of
-          one test, is being run. Note that, when the test runner program's standard error is output
+          In this case, [1/2] indicates that the first test, out of a total of
+          two test, is being run. Note that, when the test runner program's standard error is output
           to the terminal, these lines are cleared when a test succeeds.
         

-        
All 1 tests passed.

+        
Test [2/2] decltest.addOne...

+        
When the test name is an identifier, the default test runner uses the text
+          decltest instead of test.
+        

+        
All 2 tests passed.

         
This line indicates the total number of tests that have passed.

       

       {#header_open|Test Declarations#}
       

         Test declarations contain the {#link|keyword|Keyword Reference#} {#syntax#}test{#endsyntax#}, followed by an
-        optional name written as a {#link|string literal|String Literals and Unicode Code Point Literals#}, followed
-        by a {#link|block|Blocks#} containing any valid Zig code that is allowed in a {#link|function|Functions#}.
+        optional name written as a {#link|string literal|String Literals and Unicode Code Point Literals#} or an
+        {#link|identifier|Identifiers#}, followed by a {#link|block|Blocks#} containing any valid Zig code that
+        is allowed in a {#link|function|Functions#}.