motiejus/zig
1
Fork 0
Code Releases Activity

fuzzer: share zig to html rendering with autodocs

Browse Source
This commit is contained in:
Andrew Kelley
2024-07-31 23:45:43 -07:00
parent e0ffac4e3c
commit 107b272766
9 changed files with 598 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lib/compiler/std-docs.zig
View File

@@ -275,10 +275,6 @@ fn buildWasmBinary(
) ![]const u8 {
const gpa = context.gpa;
const main_src_path = try std.fs.path.join(arena, &.{
context.zig_lib_directory, "docs", "wasm", "main.zig",
});
var argv: std.ArrayListUnmanaged([]const u8) = .{};
try argv.appendSlice(arena, &.{
@@ -298,7 +294,10 @@ fn buildWasmBinary(
"--name",
"autodoc",
"-rdynamic",
main_src_path,
"--dep",
"Walk",
try std.fmt.allocPrint(arena, "-Mroot={s}/docs/wasm/main.zig", .{context.zig_lib_directory}),
try std.fmt.allocPrint(arena, "-MWalk={s}/docs/wasm/Walk.zig", .{context.zig_lib_directory}),
"--listen=-",
});

18
lib/docs/wasm/Decl.zig
View File

@@ -1,3 +1,12 @@
const Decl = @This();
const std = @import("std");
const Ast = std.zig.Ast;
const Walk = @import("Walk.zig");
const gpa = std.heap.wasm_allocator;
const assert = std.debug.assert;
const log = std.log;
const Oom = error{OutOfMemory};
ast_node: Ast.Node.Index,
file: Walk.File.Index,
/// The decl whose namespace this is in.
@@ -215,12 +224,3 @@ pub fn find(search_string: []const u8) Decl.Index {
}
return current_decl_index;
}
const Decl = @This();
const std = @import("std");
const Ast = std.zig.Ast;
const Walk = @import("Walk.zig");
const gpa = std.heap.wasm_allocator;
const assert = std.debug.assert;
const log = std.log;
const Oom = error{OutOfMemory};

20
lib/docs/wasm/Walk.zig
View File

@@ -1,4 +1,15 @@
//! Find and annotate identifiers with links to their declarations.
const Walk = @This();
const std = @import("std");
const Ast = std.zig.Ast;
const assert = std.debug.assert;
const log = std.log;
const gpa = std.heap.wasm_allocator;
const Oom = error{OutOfMemory};
pub const Decl = @import("Decl.zig");
pub var files: std.StringArrayHashMapUnmanaged(File) = .{};
pub var decls: std.ArrayListUnmanaged(Decl) = .{};
pub var modules: std.StringArrayHashMapUnmanaged(File.Index) = .{};
@@ -1120,15 +1131,6 @@ pub fn isPrimitiveNonType(name: []const u8) bool {
// try w.root();
//}
const Walk = @This();
const std = @import("std");
const Ast = std.zig.Ast;
const assert = std.debug.assert;
const Decl = @import("Decl.zig");
const log = std.log;
const gpa = std.heap.wasm_allocator;
const Oom = error{OutOfMemory};
fn shrinkToFit(m: anytype) void {
m.shrinkAndFree(gpa, m.entries.len);
}

388
lib/docs/wasm/html_render.zig Normal file
View File

@@ -0,0 +1,388 @@
const std = @import("std");
const Ast = std.zig.Ast;
const assert = std.debug.assert;
const Walk = @import("Walk");
const Decl = Walk.Decl;
const gpa = std.heap.wasm_allocator;
const Oom = error{OutOfMemory};
/// Delete this to find out where URL escaping needs to be added.
pub const missing_feature_url_escape = true;
pub const RenderSourceOptions = struct {
skip_doc_comments: bool = false,
skip_comments: bool = false,
collapse_whitespace: bool = false,
fn_link: Decl.Index = .none,
};
pub fn fileSourceHtml(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
root_node: Ast.Node.Index,
options: RenderSourceOptions,
) !void {
const ast = file_index.get_ast();
const file = file_index.get();
const g = struct {
var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
};
const token_tags = ast.tokens.items(.tag);
const token_starts = ast.tokens.items(.start);
const main_tokens = ast.nodes.items(.main_token);
const start_token = ast.firstToken(root_node);
const end_token = ast.lastToken(root_node) + 1;
var cursor: usize = token_starts[start_token];
var indent: usize = 0;
if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
for (ast.source[newline_index + 1 .. cursor]) |c| {
if (c == ' ') {
indent += 1;
} else {
break;
}
}
}
for (
token_tags[start_token..end_token],
token_starts[start_token..end_token],
start_token..,
) |tag, start, token_index| {
const between = ast.source[cursor..start];
if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
if (!options.skip_comments) {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendUnindented(out, between, indent);
try out.appendSlice(gpa, "</span>");
}
} else if (between.len > 0) {
if (options.collapse_whitespace) {
if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
try out.append(gpa, ' ');
} else {
try appendUnindented(out, between, indent);
}
}
if (tag == .eof) break;
const slice = ast.tokenSlice(token_index);
cursor = start + slice.len;
switch (tag) {
.eof => unreachable,
.keyword_addrspace,
.keyword_align,
.keyword_and,
.keyword_asm,
.keyword_async,
.keyword_await,
.keyword_break,
.keyword_catch,
.keyword_comptime,
.keyword_const,
.keyword_continue,
.keyword_defer,
.keyword_else,
.keyword_enum,
.keyword_errdefer,
.keyword_error,
.keyword_export,
.keyword_extern,
.keyword_for,
.keyword_if,
.keyword_inline,
.keyword_noalias,
.keyword_noinline,
.keyword_nosuspend,
.keyword_opaque,
.keyword_or,
.keyword_orelse,
.keyword_packed,
.keyword_anyframe,
.keyword_pub,
.keyword_resume,
.keyword_return,
.keyword_linksection,
.keyword_callconv,
.keyword_struct,
.keyword_suspend,
.keyword_switch,
.keyword_test,
.keyword_threadlocal,
.keyword_try,
.keyword_union,
.keyword_unreachable,
.keyword_usingnamespace,
.keyword_var,
.keyword_volatile,
.keyword_allowzero,
.keyword_while,
.keyword_anytype,
.keyword_fn,
=> {
try out.appendSlice(gpa, "<span class=\"tok-kw\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.string_literal,
.char_literal,
.multiline_string_literal_line,
=> {
try out.appendSlice(gpa, "<span class=\"tok-str\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.builtin => {
try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.doc_comment,
.container_doc_comment,
=> {
if (!options.skip_doc_comments) {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
}
},
.identifier => i: {
if (options.fn_link != .none) {
const fn_link = options.fn_link.get();
const fn_token = main_tokens[fn_link.ast_node];
if (token_index == fn_token + 1) {
try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
_ = missing_feature_url_escape;
try fn_link.fqn(out);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
break :i;
}
}
if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
try out.appendSlice(gpa, "<span class=\"tok-fn\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (Walk.isPrimitiveNonType(slice)) {
try out.appendSlice(gpa, "<span class=\"tok-null\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (std.zig.primitives.isPrimitive(slice)) {
try out.appendSlice(gpa, "<span class=\"tok-type\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (file.token_parents.get(token_index)) |field_access_node| {
g.field_access_buffer.clearRetainingCapacity();
try walkFieldAccesses(file_index, &g.field_access_buffer, field_access_node);
if (g.field_access_buffer.items.len > 0) {
try out.appendSlice(gpa, "<a href=\"#");
_ = missing_feature_url_escape;
try out.appendSlice(gpa, g.field_access_buffer.items);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
} else {
try appendEscaped(out, slice);
}
break :i;
}
{
g.field_access_buffer.clearRetainingCapacity();
try resolveIdentLink(file_index, &g.field_access_buffer, token_index);
if (g.field_access_buffer.items.len > 0) {
try out.appendSlice(gpa, "<a href=\"#");
_ = missing_feature_url_escape;
try out.appendSlice(gpa, g.field_access_buffer.items);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
break :i;
}
}
try appendEscaped(out, slice);
},
.number_literal => {
try out.appendSlice(gpa, "<span class=\"tok-number\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.bang,
.pipe,
.pipe_pipe,
.pipe_equal,
.equal,
.equal_equal,
.equal_angle_bracket_right,
.bang_equal,
.l_paren,
.r_paren,
.semicolon,
.percent,
.percent_equal,
.l_brace,
.r_brace,
.l_bracket,
.r_bracket,
.period,
.period_asterisk,
.ellipsis2,
.ellipsis3,
.caret,
.caret_equal,
.plus,
.plus_plus,
.plus_equal,
.plus_percent,
.plus_percent_equal,
.plus_pipe,
.plus_pipe_equal,
.minus,
.minus_equal,
.minus_percent,
.minus_percent_equal,
.minus_pipe,
.minus_pipe_equal,
.asterisk,
.asterisk_equal,
.asterisk_asterisk,
.asterisk_percent,
.asterisk_percent_equal,
.asterisk_pipe,
.asterisk_pipe_equal,
.arrow,
.colon,
.slash,
.slash_equal,
.comma,
.ampersand,
.ampersand_equal,
.question_mark,
.angle_bracket_left,
.angle_bracket_left_equal,
.angle_bracket_angle_bracket_left,
.angle_bracket_angle_bracket_left_equal,
.angle_bracket_angle_bracket_left_pipe,
.angle_bracket_angle_bracket_left_pipe_equal,
.angle_bracket_right,
.angle_bracket_right_equal,
.angle_bracket_angle_bracket_right,
.angle_bracket_angle_bracket_right_equal,
.tilde,
=> try appendEscaped(out, slice),
.invalid, .invalid_periodasterisks => return error.InvalidToken,
}
}
}
fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
var it = std.mem.splitScalar(u8, s, '\n');
var is_first_line = true;
while (it.next()) |line| {
if (is_first_line) {
try appendEscaped(out, line);
is_first_line = false;
} else {
try out.appendSlice(gpa, "\n");
try appendEscaped(out, unindent(line, indent));
}
}
}
pub fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
for (s) |c| {
try out.ensureUnusedCapacity(gpa, 6);
switch (c) {
'&' => out.appendSliceAssumeCapacity("&amp;"),
'<' => out.appendSliceAssumeCapacity("&lt;"),
'>' => out.appendSliceAssumeCapacity("&gt;"),
'"' => out.appendSliceAssumeCapacity("&quot;"),
else => out.appendAssumeCapacity(c),
}
}
}
fn walkFieldAccesses(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
node: Ast.Node.Index,
) Oom!void {
const ast = file_index.get_ast();
const node_tags = ast.nodes.items(.tag);
assert(node_tags[node] == .field_access);
const node_datas = ast.nodes.items(.data);
const main_tokens = ast.nodes.items(.main_token);
const object_node = node_datas[node].lhs;
const dot_token = main_tokens[node];
const field_ident = dot_token + 1;
switch (node_tags[object_node]) {
.identifier => {
const lhs_ident = main_tokens[object_node];
try resolveIdentLink(file_index, out, lhs_ident);
},
.field_access => {
try walkFieldAccesses(file_index, out, object_node);
},
else => {},
}
if (out.items.len > 0) {
try out.append(gpa, '.');
try out.appendSlice(gpa, ast.tokenSlice(field_ident));
}
}
fn resolveIdentLink(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
ident_token: Ast.TokenIndex,
) Oom!void {
const decl_index = file_index.get().lookup_token(ident_token);
if (decl_index == .none) return;
try resolveDeclLink(decl_index, out);
}
fn unindent(s: []const u8, indent: usize) []const u8 {
var indent_idx: usize = 0;
for (s) |c| {
if (c == ' ' and indent_idx < indent) {
indent_idx += 1;
} else {
break;
}
}
return s[indent_idx..];
}
pub fn resolveDeclLink(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
const decl = decl_index.get();
switch (decl.categorize()) {
.alias => |alias_decl| try alias_decl.get().fqn(out),
else => try decl.fqn(out),
}
}

406
lib/docs/wasm/main.zig
View File

@@ -1,15 +1,17 @@
/// Delete this to find out where URL escaping needs to be added.
const missing_feature_url_escape = true;
const gpa = std.heap.wasm_allocator;
const std = @import("std");
const log = std.log;
const assert = std.debug.assert;
const Ast = std.zig.Ast;
const Walk = @import("Walk.zig");
const Walk = @import("Walk");
const markdown = @import("markdown.zig");
const Decl = @import("Decl.zig");
const Decl = Walk.Decl;
const fileSourceHtml = @import("html_render.zig").fileSourceHtml;
const appendEscaped = @import("html_render.zig").appendEscaped;
const resolveDeclLink = @import("html_render.zig").resolveDeclLink;
const missing_feature_url_escape = @import("html_render.zig").missing_feature_url_escape;
const gpa = std.heap.wasm_allocator;
const js = struct {
extern "js" fn log(ptr: [*]const u8, len: usize) void;
@@ -439,7 +441,7 @@ fn decl_field_html_fallible(
const decl = decl_index.get();
const ast = decl.file.get_ast();
try out.appendSlice(gpa, "<pre><code>");
try file_source_html(decl.file, out, field_node, .{});
try fileSourceHtml(decl.file, out, field_node, .{});
try out.appendSlice(gpa, "</code></pre>");
const field = ast.fullContainerField(field_node).?;
@@ -478,7 +480,7 @@ fn decl_param_html_fallible(
try out.appendSlice(gpa, "<pre><code>");
try appendEscaped(out, name);
try out.appendSlice(gpa, ": ");
try file_source_html(decl.file, out, param_node, .{});
try fileSourceHtml(decl.file, out, param_node, .{});
try out.appendSlice(gpa, "</code></pre>");
if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
@@ -506,7 +508,7 @@ export fn decl_fn_proto_html(decl_index: Decl.Index, linkify_fn_name: bool) Stri
};
string_result.clearRetainingCapacity();
file_source_html(decl.file, &string_result, proto_node, .{
fileSourceHtml(decl.file, &string_result, proto_node, .{
.skip_doc_comments = true,
.skip_comments = true,
.collapse_whitespace = true,
@@ -521,7 +523,7 @@ export fn decl_source_html(decl_index: Decl.Index) String {
const decl = decl_index.get();
string_result.clearRetainingCapacity();
file_source_html(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
fatal("unable to render source: {s}", .{@errorName(err)});
};
return String.init(string_result.items);
@@ -533,7 +535,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {
return String.init("");
string_result.clearRetainingCapacity();
file_source_html(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
fileSourceHtml(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
fatal("unable to render source: {s}", .{@errorName(err)});
};
return String.init(string_result.items);
@@ -691,7 +693,7 @@ fn render_docs(
const content = doc.string(data.text.content);
if (resolve_decl_path(r.context, content)) |resolved_decl_index| {
g.link_buffer.clearRetainingCapacity();
try resolve_decl_link(resolved_decl_index, &g.link_buffer);
try resolveDeclLink(resolved_decl_index, &g.link_buffer);
try writer.writeAll("<a href=\"#");
_ = missing_feature_url_escape;
@@ -734,7 +736,7 @@ export fn decl_type_html(decl_index: Decl.Index) String {
if (ast.fullVarDecl(decl.ast_node)) |var_decl| {
if (var_decl.ast.type_node != 0) {
string_result.appendSlice(gpa, "<code>") catch @panic("OOM");
file_source_html(decl.file, &string_result, var_decl.ast.type_node, .{
fileSourceHtml(decl.file, &string_result, var_decl.ast.type_node, .{
.skip_comments = true,
.collapse_whitespace = true,
}) catch |e| {
@@ -902,382 +904,6 @@ export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Dec
return Slice(Decl.Index).init(g.members.items);
}
const RenderSourceOptions = struct {
skip_doc_comments: bool = false,
skip_comments: bool = false,
collapse_whitespace: bool = false,
fn_link: Decl.Index = .none,
};
fn file_source_html(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
root_node: Ast.Node.Index,
options: RenderSourceOptions,
) !void {
const ast = file_index.get_ast();
const file = file_index.get();
const g = struct {
var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
};
const token_tags = ast.tokens.items(.tag);
const token_starts = ast.tokens.items(.start);
const main_tokens = ast.nodes.items(.main_token);
const start_token = ast.firstToken(root_node);
const end_token = ast.lastToken(root_node) + 1;
var cursor: usize = token_starts[start_token];
var indent: usize = 0;
if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
for (ast.source[newline_index + 1 .. cursor]) |c| {
if (c == ' ') {
indent += 1;
} else {
break;
}
}
}
for (
token_tags[start_token..end_token],
token_starts[start_token..end_token],
start_token..,
) |tag, start, token_index| {
const between = ast.source[cursor..start];
if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
if (!options.skip_comments) {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendUnindented(out, between, indent);
try out.appendSlice(gpa, "</span>");
}
} else if (between.len > 0) {
if (options.collapse_whitespace) {
if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
try out.append(gpa, ' ');
} else {
try appendUnindented(out, between, indent);
}
}
if (tag == .eof) break;
const slice = ast.tokenSlice(token_index);
cursor = start + slice.len;
switch (tag) {
.eof => unreachable,
.keyword_addrspace,
.keyword_align,
.keyword_and,
.keyword_asm,
.keyword_async,
.keyword_await,
.keyword_break,
.keyword_catch,
.keyword_comptime,
.keyword_const,
.keyword_continue,
.keyword_defer,
.keyword_else,
.keyword_enum,
.keyword_errdefer,
.keyword_error,
.keyword_export,
.keyword_extern,
.keyword_for,
.keyword_if,
.keyword_inline,
.keyword_noalias,
.keyword_noinline,
.keyword_nosuspend,
.keyword_opaque,
.keyword_or,
.keyword_orelse,
.keyword_packed,
.keyword_anyframe,
.keyword_pub,
.keyword_resume,
.keyword_return,
.keyword_linksection,
.keyword_callconv,
.keyword_struct,
.keyword_suspend,
.keyword_switch,
.keyword_test,
.keyword_threadlocal,
.keyword_try,
.keyword_union,
.keyword_unreachable,
.keyword_usingnamespace,
.keyword_var,
.keyword_volatile,
.keyword_allowzero,
.keyword_while,
.keyword_anytype,
.keyword_fn,
=> {
try out.appendSlice(gpa, "<span class=\"tok-kw\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.string_literal,
.char_literal,
.multiline_string_literal_line,
=> {
try out.appendSlice(gpa, "<span class=\"tok-str\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.builtin => {
try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.doc_comment,
.container_doc_comment,
=> {
if (!options.skip_doc_comments) {
try out.appendSlice(gpa, "<span class=\"tok-comment\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
}
},
.identifier => i: {
if (options.fn_link != .none) {
const fn_link = options.fn_link.get();
const fn_token = main_tokens[fn_link.ast_node];
if (token_index == fn_token + 1) {
try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
_ = missing_feature_url_escape;
try fn_link.fqn(out);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
break :i;
}
}
if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
try out.appendSlice(gpa, "<span class=\"tok-fn\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (Walk.isPrimitiveNonType(slice)) {
try out.appendSlice(gpa, "<span class=\"tok-null\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (std.zig.primitives.isPrimitive(slice)) {
try out.appendSlice(gpa, "<span class=\"tok-type\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
break :i;
}
if (file.token_parents.get(token_index)) |field_access_node| {
g.field_access_buffer.clearRetainingCapacity();
try walk_field_accesses(file_index, &g.field_access_buffer, field_access_node);
if (g.field_access_buffer.items.len > 0) {
try out.appendSlice(gpa, "<a href=\"#");
_ = missing_feature_url_escape;
try out.appendSlice(gpa, g.field_access_buffer.items);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
} else {
try appendEscaped(out, slice);
}
break :i;
}
{
g.field_access_buffer.clearRetainingCapacity();
try resolve_ident_link(file_index, &g.field_access_buffer, token_index);
if (g.field_access_buffer.items.len > 0) {
try out.appendSlice(gpa, "<a href=\"#");
_ = missing_feature_url_escape;
try out.appendSlice(gpa, g.field_access_buffer.items);
try out.appendSlice(gpa, "\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</a>");
break :i;
}
}
try appendEscaped(out, slice);
},
.number_literal => {
try out.appendSlice(gpa, "<span class=\"tok-number\">");
try appendEscaped(out, slice);
try out.appendSlice(gpa, "</span>");
},
.bang,
.pipe,
.pipe_pipe,
.pipe_equal,
.equal,
.equal_equal,
.equal_angle_bracket_right,
.bang_equal,
.l_paren,
.r_paren,
.semicolon,
.percent,
.percent_equal,
.l_brace,
.r_brace,
.l_bracket,
.r_bracket,
.period,
.period_asterisk,
.ellipsis2,
.ellipsis3,
.caret,
.caret_equal,
.plus,
.plus_plus,
.plus_equal,
.plus_percent,
.plus_percent_equal,
.plus_pipe,
.plus_pipe_equal,
.minus,
.minus_equal,
.minus_percent,
.minus_percent_equal,
.minus_pipe,
.minus_pipe_equal,
.asterisk,
.asterisk_equal,
.asterisk_asterisk,
.asterisk_percent,
.asterisk_percent_equal,
.asterisk_pipe,
.asterisk_pipe_equal,
.arrow,
.colon,
.slash,
.slash_equal,
.comma,
.ampersand,
.ampersand_equal,
.question_mark,
.angle_bracket_left,
.angle_bracket_left_equal,
.angle_bracket_angle_bracket_left,
.angle_bracket_angle_bracket_left_equal,
.angle_bracket_angle_bracket_left_pipe,
.angle_bracket_angle_bracket_left_pipe_equal,
.angle_bracket_right,
.angle_bracket_right_equal,
.angle_bracket_angle_bracket_right,
.angle_bracket_angle_bracket_right_equal,
.tilde,
=> try appendEscaped(out, slice),
.invalid, .invalid_periodasterisks => return error.InvalidToken,
}
}
}
fn unindent(s: []const u8, indent: usize) []const u8 {
var indent_idx: usize = 0;
for (s) |c| {
if (c == ' ' and indent_idx < indent) {
indent_idx += 1;
} else {
break;
}
}
return s[indent_idx..];
}
fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
var it = std.mem.splitScalar(u8, s, '\n');
var is_first_line = true;
while (it.next()) |line| {
if (is_first_line) {
try appendEscaped(out, line);
is_first_line = false;
} else {
try out.appendSlice(gpa, "\n");
try appendEscaped(out, unindent(line, indent));
}
}
}
fn resolve_ident_link(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
ident_token: Ast.TokenIndex,
) Oom!void {
const decl_index = file_index.get().lookup_token(ident_token);
if (decl_index == .none) return;
try resolve_decl_link(decl_index, out);
}
fn resolve_decl_link(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
const decl = decl_index.get();
switch (decl.categorize()) {
.alias => |alias_decl| try alias_decl.get().fqn(out),
else => try decl.fqn(out),
}
}
fn walk_field_accesses(
file_index: Walk.File.Index,
out: *std.ArrayListUnmanaged(u8),
node: Ast.Node.Index,
) Oom!void {
const ast = file_index.get_ast();
const node_tags = ast.nodes.items(.tag);
assert(node_tags[node] == .field_access);
const node_datas = ast.nodes.items(.data);
const main_tokens = ast.nodes.items(.main_token);
const object_node = node_datas[node].lhs;
const dot_token = main_tokens[node];
const field_ident = dot_token + 1;
switch (node_tags[object_node]) {
.identifier => {
const lhs_ident = main_tokens[object_node];
try resolve_ident_link(file_index, out, lhs_ident);
},
.field_access => {
try walk_field_accesses(file_index, out, object_node);
},
else => {},
}
if (out.items.len > 0) {
try out.append(gpa, '.');
try out.appendSlice(gpa, ast.tokenSlice(field_ident));
}
}
fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
for (s) |c| {
try out.ensureUnusedCapacity(gpa, 6);
switch (c) {
'&' => out.appendSliceAssumeCapacity("&amp;"),
'<' => out.appendSliceAssumeCapacity("&lt;"),
'>' => out.appendSliceAssumeCapacity("&gt;"),
'"' => out.appendSliceAssumeCapacity("&quot;"),
else => out.appendAssumeCapacity(c),
}
}
}
fn count_scalar(haystack: []const u8, needle: u8) usize {
var total: usize = 0;
for (haystack) |elem| {

60
lib/fuzzer/index.html
View File

@@ -2,12 +2,56 @@
<html>
<head>
<meta charset="utf-8">
<title>Zig Documentation</title>
<title>Zig Build System Interface</title>
<style type="text/css">
body {
font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif;
color: #000000;
}
.hidden {
display: none;
}
table {
width: 100%;
}
a {
color: #2A6286;
}
pre{
font-family:"Source Code Pro",monospace;
font-size:1em;
background-color:#F5F5F5;
padding: 1em;
margin: 0;
overflow-x: auto;
}
:not(pre) > code {
white-space: break-spaces;
}
code {
font-family:"Source Code Pro",monospace;
font-size: 0.9em;
}
code a {
color: #000000;
}
kbd {
color: #000;
background-color: #fafbfc;
border-color: #d1d5da;
border-bottom-color: #c6cbd1;
box-shadow-color: #c6cbd1;
display: inline-block;
padding: 0.3em 0.2em;
font: 1.2em monospace;
line-height: 0.8em;
vertical-align: middle;
border: solid 1px;
border-radius: 3px;
box-shadow: inset 0 -1px 0;
cursor: default;
}
.tok-kw {
color: #333;
font-weight: bold;
@@ -42,6 +86,16 @@
background-color: #111;
color: #bbb;
}
pre {
background-color: #222;
color: #ccc;
}
a {
color: #88f;
}
code a {
color: #ccc;
}
.tok-kw {
color: #eee;
}
@@ -70,6 +124,10 @@
</style>
</head>
<body>
<div id="sectSource" class="hidden">
<h2>Source Code</h2>
<pre><code id="sourceText"></code></pre>
</div>
<script src="main.js"></script>
</body>
</html>

48
lib/fuzzer/main.js
View File

@@ -1,4 +1,7 @@
(function() {
const domSectSource = document.getElementById("sectSource");
const domSourceText = document.getElementById("sourceText");
let wasm_promise = fetch("main.wasm");
let sources_promise = fetch("sources.tar").then(function(response) {
if (!response.ok) throw new Error("unable to download sources");
@@ -30,11 +33,56 @@
const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
wasm_array.set(js_array);
wasm_exports.unpack(ptr, js_array.length);
render();
});
});
function render() {
domSectSource.classList.add("hidden");
// TODO this is temporary debugging data
renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
}
function renderSource(path) {
const decl_index = findFileRoot(path);
if (decl_index == null) throw new Error("file not found: " + path);
const h2 = domSectSource.children[0];
h2.innerText = path;
domSourceText.innerHTML = declSourceHtml(decl_index);
domSectSource.classList.remove("hidden");
}
function findFileRoot(path) {
setInputString(path);
const result = wasm_exports.find_file_root();
if (result === -1) return null;
return result;
}
function decodeString(ptr, len) {
if (len === 0) return "";
return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
}
function setInputString(s) {
const jsArray = text_encoder.encode(s);
const len = jsArray.length;
const ptr = wasm_exports.set_input_string(len);
const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
wasmArray.set(jsArray);
}
function declSourceHtml(decl_index) {
return unwrapString(wasm_exports.decl_source_html(decl_index));
}
function unwrapString(bigint) {
const ptr = Number(bigint & 0xffffffffn);
const len = Number(bigint >> 32n);
return decodeString(ptr, len);
}
})();

44
lib/fuzzer/wasm/main.zig
View File

@@ -2,6 +2,8 @@ const std = @import("std");
const assert = std.debug.assert;
const Walk = @import("Walk");
const Decl = Walk.Decl;
const html_render = @import("html_render");
const gpa = std.heap.wasm_allocator;
const log = std.log;
@@ -52,6 +54,48 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
};
}
/// Set by `set_input_string`.
var input_string: std.ArrayListUnmanaged(u8) = .{};
var string_result: std.ArrayListUnmanaged(u8) = .{};
export fn set_input_string(len: usize) [*]u8 {
input_string.resize(gpa, len) catch @panic("OOM");
return input_string.items.ptr;
}
/// Looks up the root struct decl corresponding to a file by path.
/// Uses `input_string`.
export fn find_file_root() Decl.Index {
const file: Walk.File.Index = @enumFromInt(Walk.files.getIndex(input_string.items) orelse return .none);
return file.findRootDecl();
}
export fn decl_source_html(decl_index: Decl.Index) String {
const decl = decl_index.get();
string_result.clearRetainingCapacity();
html_render.fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
fatal("unable to render source: {s}", .{@errorName(err)});
};
return String.init(string_result.items);
}
const String = Slice(u8);
fn Slice(T: type) type {
return packed struct(u64) {
ptr: u32,
len: u32,
fn init(s: []const T) @This() {
return .{
.ptr = @intFromPtr(s.ptr),
.len = s.len,
};
}
};
}
fn unpackInner(tar_bytes: []u8) !void {
var fbs = std.io.fixedBufferStream(tar_bytes);
var file_name_buffer: [1024]u8 = undefined;

39
lib/std/Build/Fuzz.zig
View File

@@ -235,30 +235,29 @@ pub const WebServer = struct {
.root_dir = ws.zig_lib_directory,
.sub_path = "docs/wasm/Walk.zig",
};
const html_render_src_path: Build.Cache.Path = .{
.root_dir = ws.zig_lib_directory,
.sub_path = "docs/wasm/html_render.zig",
};
var argv: std.ArrayListUnmanaged([]const u8) = .{};
try argv.appendSlice(arena, &.{
ws.zig_exe_path,
"build-exe",
"-fno-entry",
"-O",
@tagName(optimize_mode),
"-target",
"wasm32-freestanding",
"-mcpu",
"baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext",
"--cache-dir",
ws.global_cache_directory.path orelse ".",
"--global-cache-dir",
ws.global_cache_directory.path orelse ".",
"--name",
"fuzzer",
"-rdynamic",
"--dep",
"Walk",
try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}),
try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}),
ws.zig_exe_path, "build-exe", //
"-fno-entry", //
"-O", @tagName(optimize_mode), //
"-target", "wasm32-freestanding", //
"-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
"--cache-dir", ws.global_cache_directory.path orelse ".", //
"--global-cache-dir", ws.global_cache_directory.path orelse ".", //
"--name", "fuzzer", //
"-rdynamic", //
"--dep", "Walk", //
"--dep", "html_render", //
try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
"--dep", "Walk", //
try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
"--listen=-",
});