motiejus/zig
1
Fork 0
Code Releases Activity

(breaking) rework stream abstractions

Browse Source 
The main goal here is to make the function pointers comptime, so that we
don't have to do the crazy stuff with async function frames.

Since InStream, OutStream, and SeekableStream are already generic
across error sets, it's not really worse to make them generic across the
vtable as well.

See #764 for the open issue acknowledging that using generics for these
abstractions is a design flaw.

See #130 for the efforts to make these abstractions non-generic.

This commit also changes the OutStream API so that `write` returns
number of bytes written, and `writeAll` is the one that loops until the
whole buffer is written.
This commit is contained in:
Andrew Kelley
2020-03-10 15:27:45 -04:00
parent 1ad831a0ef
commit ba0e3be5cf
24 changed files with 748 additions and 772 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/std/zig/ast.zig
View File

@@ -375,7 +375,7 @@ pub const Error = union(enum) {
token: TokenIndex,
pub fn render(self: *const ThisError, tokens: *Tree.TokenList, stream: var) !void {
return stream.write(msg);
return stream.writeAll(msg);
}
};
}

139
lib/std/zig/render.zig
View File

@@ -12,64 +12,58 @@ pub const Error = error{
};
/// Returns whether anything changed
pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Child.Error || Error)!bool {
comptime assert(@typeInfo(@TypeOf(stream)) == .Pointer);
var anything_changed: bool = false;
pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Error || Error)!bool {
// make a passthrough stream that checks whether something changed
const MyStream = struct {
const MyStream = @This();
const StreamError = @TypeOf(stream).Child.Error;
const Stream = std.io.OutStream(StreamError);
const StreamError = @TypeOf(stream).Error;
anything_changed_ptr: *bool,
child_stream: @TypeOf(stream),
stream: Stream,
anything_changed: bool,
source_index: usize,
source: []const u8,
fn write(iface_stream: *Stream, bytes: []const u8) StreamError!usize {
const self = @fieldParentPtr(MyStream, "stream", iface_stream);
if (!self.anything_changed_ptr.*) {
fn write(self: *MyStream, bytes: []const u8) StreamError!usize {
if (!self.anything_changed) {
const end = self.source_index + bytes.len;
if (end > self.source.len) {
self.anything_changed_ptr.* = true;
self.anything_changed = true;
} else {
const src_slice = self.source[self.source_index..end];
self.source_index += bytes.len;
if (!mem.eql(u8, bytes, src_slice)) {
self.anything_changed_ptr.* = true;
self.anything_changed = true;
}
}
}
return self.child_stream.writeOnce(bytes);
return self.child_stream.write(bytes);
}
};
var my_stream = MyStream{
.stream = MyStream.Stream{ .writeFn = MyStream.write },
.child_stream = stream,
.anything_changed_ptr = &anything_changed,
.anything_changed = false,
.source_index = 0,
.source = tree.source,
};
const my_stream_stream: std.io.OutStream(*MyStream, MyStream.StreamError, MyStream.write) = .{
.context = &my_stream,
};
try renderRoot(allocator, &my_stream.stream, tree);
try renderRoot(allocator, my_stream_stream, tree);
if (!anything_changed and my_stream.source_index != my_stream.source.len) {
anything_changed = true;
if (my_stream.source_index != my_stream.source.len) {
my_stream.anything_changed = true;
}
return anything_changed;
return my_stream.anything_changed;
}
fn renderRoot(
allocator: *mem.Allocator,
stream: var,
tree: *ast.Tree,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
var tok_it = tree.tokens.iterator(0);
// render all the line comments at the beginning of the file
@@ -189,7 +183,7 @@ fn renderRoot(
}
}
fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *ast.Node) @TypeOf(stream).Child.Error!void {
fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *ast.Node) @TypeOf(stream).Error!void {
const first_token = node.firstToken();
var prev_token = first_token;
if (prev_token == 0) return;
@@ -204,11 +198,11 @@ fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *as
}
}
fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node) (@TypeOf(stream).Child.Error || Error)!void {
fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node) (@TypeOf(stream).Error || Error)!void {
try renderContainerDecl(allocator, stream, tree, indent, start_col, decl, .Newline);
}
fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node, space: Space) (@TypeOf(stream).Child.Error || Error)!void {
fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node, space: Space) (@TypeOf(stream).Error || Error)!void {
switch (decl.id) {
.FnProto => {
const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl);
@@ -343,7 +337,7 @@ fn renderExpression(
start_col: *usize,
base: *ast.Node,
space: Space,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
switch (base.id) {
.Identifier => {
const identifier = @fieldParentPtr(ast.Node.Identifier, "base", base);
@@ -449,9 +443,9 @@ fn renderExpression(
switch (op_tok_id) {
.Asterisk, .AsteriskAsterisk => try stream.writeByte('*'),
.LBracket => if (tree.tokens.at(prefix_op_node.op_token + 2).id == .Identifier)
try stream.write("[*c")
try stream.writeAll("[*c")
else
try stream.write("[*"),
try stream.writeAll("[*"),
else => unreachable,
}
if (ptr_info.sentinel) |sentinel| {
@@ -757,7 +751,7 @@ fn renderExpression(
while (it.next()) |field_init| {
var find_stream = FindByteOutStream.init('\n');
var dummy_col: usize = 0;
try renderExpression(allocator, &find_stream.stream, tree, 0, &dummy_col, field_init.*, Space.None);
try renderExpression(allocator, find_stream.outStream(), tree, 0, &dummy_col, field_init.*, Space.None);
if (find_stream.byte_found) break :blk false;
}
break :blk true;
@@ -909,8 +903,7 @@ fn renderExpression(
var column_widths = widths[widths.len - row_size ..];
// Null stream for counting the printed length of each expression
var null_stream = std.io.NullOutStream.init();
var counting_stream = std.io.CountingOutStream(std.io.NullOutStream.Error).init(&null_stream.stream);
var counting_stream = std.io.CountingOutStream(@TypeOf(std.io.null_out_stream)).init(std.io.null_out_stream);
var it = exprs.iterator(0);
var i: usize = 0;
@@ -918,7 +911,7 @@ fn renderExpression(
while (it.next()) |expr| : (i += 1) {
counting_stream.bytes_written = 0;
var dummy_col: usize = 0;
try renderExpression(allocator, &counting_stream.stream, tree, indent, &dummy_col, expr.*, Space.None);
try renderExpression(allocator, counting_stream.outStream(), tree, indent, &dummy_col, expr.*, Space.None);
const width = @intCast(usize, counting_stream.bytes_written);
const col = i % row_size;
column_widths[col] = std.math.max(column_widths[col], width);
@@ -1336,7 +1329,7 @@ fn renderExpression(
// TODO: Remove condition after deprecating 'typeOf'. See https://github.com/ziglang/zig/issues/1348
if (mem.eql(u8, tree.tokenSlicePtr(tree.tokens.at(builtin_call.builtin_token)), "@typeOf")) {
try stream.write("@TypeOf");
try stream.writeAll("@TypeOf");
} else {
try renderToken(tree, stream, builtin_call.builtin_token, indent, start_col, Space.None); // @name
}
@@ -1505,9 +1498,9 @@ fn renderExpression(
try renderExpression(allocator, stream, tree, indent, start_col, callconv_expr, Space.None);
try renderToken(tree, stream, callconv_rparen, indent, start_col, Space.Space); // )
} else if (cc_rewrite_str) |str| {
try stream.write("callconv(");
try stream.write(mem.toSliceConst(u8, str));
try stream.write(") ");
try stream.writeAll("callconv(");
try stream.writeAll(mem.toSliceConst(u8, str));
try stream.writeAll(") ");
}
switch (fn_proto.return_type) {
@@ -1997,11 +1990,11 @@ fn renderExpression(
.AsmInput => {
const asm_input = @fieldParentPtr(ast.Node.AsmInput, "base", base);
try stream.write("[");
try stream.writeAll("[");
try renderExpression(allocator, stream, tree, indent, start_col, asm_input.symbolic_name, Space.None);
try stream.write("] ");
try stream.writeAll("] ");
try renderExpression(allocator, stream, tree, indent, start_col, asm_input.constraint, Space.None);
try stream.write(" (");
try stream.writeAll(" (");
try renderExpression(allocator, stream, tree, indent, start_col, asm_input.expr, Space.None);
return renderToken(tree, stream, asm_input.lastToken(), indent, start_col, space); // )
},
@@ -2009,18 +2002,18 @@ fn renderExpression(
.AsmOutput => {
const asm_output = @fieldParentPtr(ast.Node.AsmOutput, "base", base);
try stream.write("[");
try stream.writeAll("[");
try renderExpression(allocator, stream, tree, indent, start_col, asm_output.symbolic_name, Space.None);
try stream.write("] ");
try stream.writeAll("] ");
try renderExpression(allocator, stream, tree, indent, start_col, asm_output.constraint, Space.None);
try stream.write(" (");
try stream.writeAll(" (");
switch (asm_output.kind) {
ast.Node.AsmOutput.Kind.Variable => |variable_name| {
try renderExpression(allocator, stream, tree, indent, start_col, &variable_name.base, Space.None);
},
ast.Node.AsmOutput.Kind.Return => |return_type| {
try stream.write("-> ");
try stream.writeAll("-> ");
try renderExpression(allocator, stream, tree, indent, start_col, return_type, Space.None);
},
}
@@ -2052,7 +2045,7 @@ fn renderVarDecl(
indent: usize,
start_col: *usize,
var_decl: *ast.Node.VarDecl,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
if (var_decl.visib_token) |visib_token| {
try renderToken(tree, stream, visib_token, indent, start_col, Space.Space); // pub
}
@@ -2125,7 +2118,7 @@ fn renderParamDecl(
start_col: *usize,
base: *ast.Node,
space: Space,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
const param_decl = @fieldParentPtr(ast.Node.ParamDecl, "base", base);
try renderDocComments(tree, stream, param_decl, indent, start_col);
@@ -2154,7 +2147,7 @@ fn renderStatement(
indent: usize,
start_col: *usize,
base: *ast.Node,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
switch (base.id) {
.VarDecl => {
const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", base);
@@ -2193,7 +2186,7 @@ fn renderTokenOffset(
start_col: *usize,
space: Space,
token_skip_bytes: usize,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
if (space == Space.BlockStart) {
if (start_col.* < indent + indent_delta)
return renderToken(tree, stream, token_index, indent, start_col, Space.Space);
@@ -2204,7 +2197,7 @@ fn renderTokenOffset(
}
var token = tree.tokens.at(token_index);
try stream.write(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " "));
try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " "));
if (space == Space.NoComment)
return;
@@ -2214,15 +2207,15 @@ fn renderTokenOffset(
if (space == Space.Comma) switch (next_token.id) {
.Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline),
.LineComment => {
try stream.write(", ");
try stream.writeAll(", ");
return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline);
},
else => {
if (token_index + 2 < tree.tokens.len and tree.tokens.at(token_index + 2).id == .MultilineStringLiteralLine) {
try stream.write(",");
try stream.writeAll(",");
return;
} else {
try stream.write(",\n");
try stream.writeAll(",\n");
start_col.* = 0;
return;
}
@@ -2246,7 +2239,7 @@ fn renderTokenOffset(
if (next_token.id == .MultilineStringLiteralLine) {
return;
} else {
try stream.write("\n");
try stream.writeAll("\n");
start_col.* = 0;
return;
}
@@ -2309,7 +2302,7 @@ fn renderTokenOffset(
if (next_token.id == .MultilineStringLiteralLine) {
return;
} else {
try stream.write("\n");
try stream.writeAll("\n");
start_col.* = 0;
return;
}
@@ -2327,7 +2320,7 @@ fn renderTokenOffset(
const newline_count = if (loc.line == 1) @as(u8, 1) else @as(u8, 2);
try stream.writeByteNTimes('\n', newline_count);
try stream.writeByteNTimes(' ', indent);
try stream.write(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " "));
try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " "));
offset += 1;
token = next_token;
@@ -2338,7 +2331,7 @@ fn renderTokenOffset(
if (next_token.id == .MultilineStringLiteralLine) {
return;
} else {
try stream.write("\n");
try stream.writeAll("\n");
start_col.* = 0;
return;
}
@@ -2381,7 +2374,7 @@ fn renderToken(
indent: usize,
start_col: *usize,
space: Space,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
return renderTokenOffset(tree, stream, token_index, indent, start_col, space, 0);
}
@@ -2391,7 +2384,7 @@ fn renderDocComments(
node: var,
indent: usize,
start_col: *usize,
) (@TypeOf(stream).Child.Error || Error)!void {
) (@TypeOf(stream).Error || Error)!void {
const comment = node.doc_comments orelse return;
var it = comment.lines.iterator(0);
const first_token = node.firstToken();
@@ -2401,7 +2394,7 @@ fn renderDocComments(
try stream.writeByteNTimes(' ', indent);
} else {
try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.NoComment);
try stream.write("\n");
try stream.writeAll("\n");
try stream.writeByteNTimes(' ', indent);
}
}
@@ -2427,27 +2420,23 @@ fn nodeCausesSliceOpSpace(base: *ast.Node) bool {
};
}
// An OutStream that returns whether the given character has been written to it.
// The contents are not written to anything.
/// A `std.io.OutStream` that returns whether the given character has been written to it.
/// The contents are not written to anything.
const FindByteOutStream = struct {
const Self = FindByteOutStream;
pub const Error = error{};
pub const Stream = std.io.OutStream(Error);
stream: Stream,
byte_found: bool,
byte: u8,
pub fn init(byte: u8) Self {
return Self{
.stream = Stream{ .writeFn = writeFn },
pub const Error = error{};
pub const OutStream = std.io.OutStream(*FindByteOutStream, Error, write);
pub fn init(byte: u8) FindByteOutStream {
return FindByteOutStream{
.byte = byte,
.byte_found = false,
};
}
fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize {
const self = @fieldParentPtr(Self, "stream", out_stream);
pub fn write(self: *FindByteOutStream, bytes: []const u8) Error!usize {
if (self.byte_found) return bytes.len;
self.byte_found = blk: {
for (bytes) |b|
@@ -2456,11 +2445,15 @@ const FindByteOutStream = struct {
};
return bytes.len;
}
pub fn outStream(self: *FindByteOutStream) OutStream {
return .{ .context = self };
}
};
fn copyFixingWhitespace(stream: var, slice: []const u8) @TypeOf(stream).Child.Error!void {
fn copyFixingWhitespace(stream: var, slice: []const u8) @TypeOf(stream).Error!void {
for (slice) |byte| switch (byte) {
'\t' => try stream.write(" "),
'\t' => try stream.writeAll(" "),
'\r' => {},
else => try stream.writeByte(byte),
};