commit 8ea8cff4912bf66b8e95b7518877c7f55c7a9356 (tree)
parent 111d3792e0f2d92f7b9f73fb8265ee17fc229eb4
Author: Andrew Kelley <andrew@ziglang.org>
Date: Fri, 19 Jul 2019 17:54:06 -0400
slightly simpler implementation
Diffstat:
4 files changed, 135 insertions(+), 141 deletions(-)
diff --git a/doc/docgen.zig b/doc/docgen.zig
@@ -742,101 +742,101 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
const token = tokenizer.next();
try writeEscaped(out, src[index..token.start]);
switch (token.id) {
- std.zig.Token.Id.Eof => break,
-
- std.zig.Token.Id.Keyword_align,
- std.zig.Token.Id.Keyword_and,
- std.zig.Token.Id.Keyword_asm,
- std.zig.Token.Id.Keyword_async,
- std.zig.Token.Id.Keyword_await,
- std.zig.Token.Id.Keyword_break,
- std.zig.Token.Id.Keyword_cancel,
- std.zig.Token.Id.Keyword_catch,
- std.zig.Token.Id.Keyword_comptime,
- std.zig.Token.Id.Keyword_const,
- std.zig.Token.Id.Keyword_continue,
- std.zig.Token.Id.Keyword_defer,
- std.zig.Token.Id.Keyword_else,
- std.zig.Token.Id.Keyword_enum,
- std.zig.Token.Id.Keyword_errdefer,
- std.zig.Token.Id.Keyword_error,
- std.zig.Token.Id.Keyword_export,
- std.zig.Token.Id.Keyword_extern,
- std.zig.Token.Id.Keyword_for,
- std.zig.Token.Id.Keyword_if,
- std.zig.Token.Id.Keyword_inline,
- std.zig.Token.Id.Keyword_nakedcc,
- std.zig.Token.Id.Keyword_noalias,
- std.zig.Token.Id.Keyword_or,
- std.zig.Token.Id.Keyword_orelse,
- std.zig.Token.Id.Keyword_packed,
- std.zig.Token.Id.Keyword_promise,
- std.zig.Token.Id.Keyword_pub,
- std.zig.Token.Id.Keyword_resume,
- std.zig.Token.Id.Keyword_return,
- std.zig.Token.Id.Keyword_linksection,
- std.zig.Token.Id.Keyword_stdcallcc,
- std.zig.Token.Id.Keyword_struct,
- std.zig.Token.Id.Keyword_suspend,
- std.zig.Token.Id.Keyword_switch,
- std.zig.Token.Id.Keyword_test,
- std.zig.Token.Id.Keyword_threadlocal,
- std.zig.Token.Id.Keyword_try,
- std.zig.Token.Id.Keyword_union,
- std.zig.Token.Id.Keyword_unreachable,
- std.zig.Token.Id.Keyword_usingnamespace,
- std.zig.Token.Id.Keyword_var,
- std.zig.Token.Id.Keyword_volatile,
- std.zig.Token.Id.Keyword_allowzero,
- std.zig.Token.Id.Keyword_while,
+ .Eof => break,
+
+ .Keyword_align,
+ .Keyword_and,
+ .Keyword_asm,
+ .Keyword_async,
+ .Keyword_await,
+ .Keyword_break,
+ .Keyword_cancel,
+ .Keyword_catch,
+ .Keyword_comptime,
+ .Keyword_const,
+ .Keyword_continue,
+ .Keyword_defer,
+ .Keyword_else,
+ .Keyword_enum,
+ .Keyword_errdefer,
+ .Keyword_error,
+ .Keyword_export,
+ .Keyword_extern,
+ .Keyword_for,
+ .Keyword_if,
+ .Keyword_inline,
+ .Keyword_nakedcc,
+ .Keyword_noalias,
+ .Keyword_or,
+ .Keyword_orelse,
+ .Keyword_packed,
+ .Keyword_promise,
+ .Keyword_pub,
+ .Keyword_resume,
+ .Keyword_return,
+ .Keyword_linksection,
+ .Keyword_stdcallcc,
+ .Keyword_struct,
+ .Keyword_suspend,
+ .Keyword_switch,
+ .Keyword_test,
+ .Keyword_threadlocal,
+ .Keyword_try,
+ .Keyword_union,
+ .Keyword_unreachable,
+ .Keyword_usingnamespace,
+ .Keyword_var,
+ .Keyword_volatile,
+ .Keyword_allowzero,
+ .Keyword_while,
=> {
try out.write("<span class=\"tok-kw\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
},
- std.zig.Token.Id.Keyword_fn => {
+ .Keyword_fn => {
try out.write("<span class=\"tok-kw\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
next_tok_is_fn = true;
},
- std.zig.Token.Id.Keyword_undefined,
- std.zig.Token.Id.Keyword_null,
- std.zig.Token.Id.Keyword_true,
- std.zig.Token.Id.Keyword_false,
+ .Keyword_undefined,
+ .Keyword_null,
+ .Keyword_true,
+ .Keyword_false,
=> {
try out.write("<span class=\"tok-null\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
},
- std.zig.Token.Id.StringLiteral,
- std.zig.Token.Id.MultilineStringLiteralLine,
- std.zig.Token.Id.CharLiteral,
+ .StringLiteral,
+ .MultilineStringLiteralLine,
+ .CharLiteral,
=> {
try out.write("<span class=\"tok-str\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
},
- std.zig.Token.Id.Builtin => {
+ .Builtin => {
try out.write("<span class=\"tok-builtin\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
},
- std.zig.Token.Id.LineComment,
- std.zig.Token.Id.DocComment,
- std.zig.Token.Id.ShebangLine,
+ .LineComment,
+ .DocComment,
+ .ShebangLine,
=> {
try out.write("<span class=\"tok-comment\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
},
- std.zig.Token.Id.Identifier => {
+ .Identifier => {
if (prev_tok_was_fn) {
try out.write("<span class=\"tok-fn\">");
try writeEscaped(out, src[token.start..token.end]);
@@ -864,74 +864,72 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok
}
},
- std.zig.Token.Id.IntegerLiteral,
- std.zig.Token.Id.FloatLiteral,
+ .IntegerLiteral,
+ .FloatLiteral,
=> {
try out.write("<span class=\"tok-number\">");
try writeEscaped(out, src[token.start..token.end]);
try out.write("</span>");
},
- std.zig.Token.Id.Bang,
- std.zig.Token.Id.Pipe,
- std.zig.Token.Id.PipePipe,
- std.zig.Token.Id.PipeEqual,
- std.zig.Token.Id.Equal,
- std.zig.Token.Id.EqualEqual,
- std.zig.Token.Id.EqualAngleBracketRight,
- std.zig.Token.Id.BangEqual,
- std.zig.Token.Id.LParen,
- std.zig.Token.Id.RParen,
- std.zig.Token.Id.Semicolon,
- std.zig.Token.Id.Percent,
- std.zig.Token.Id.PercentEqual,
- std.zig.Token.Id.LBrace,
- std.zig.Token.Id.RBrace,
- std.zig.Token.Id.LBracket,
- std.zig.Token.Id.RBracket,
- std.zig.Token.Id.Period,
- std.zig.Token.Id.Ellipsis2,
- std.zig.Token.Id.Ellipsis3,
- std.zig.Token.Id.Caret,
- std.zig.Token.Id.CaretEqual,
- std.zig.Token.Id.Plus,
- std.zig.Token.Id.PlusPlus,
- std.zig.Token.Id.PlusEqual,
- std.zig.Token.Id.PlusPercent,
- std.zig.Token.Id.PlusPercentEqual,
- std.zig.Token.Id.Minus,
- std.zig.Token.Id.MinusEqual,
- std.zig.Token.Id.MinusPercent,
- std.zig.Token.Id.MinusPercentEqual,
- std.zig.Token.Id.Asterisk,
- std.zig.Token.Id.AsteriskEqual,
- std.zig.Token.Id.AsteriskAsterisk,
- std.zig.Token.Id.AsteriskPercent,
- std.zig.Token.Id.AsteriskPercentEqual,
- std.zig.Token.Id.Arrow,
- std.zig.Token.Id.Colon,
- std.zig.Token.Id.Slash,
- std.zig.Token.Id.SlashEqual,
- std.zig.Token.Id.Comma,
- std.zig.Token.Id.Ampersand,
- std.zig.Token.Id.AmpersandEqual,
- std.zig.Token.Id.QuestionMark,
- std.zig.Token.Id.AngleBracketLeft,
- std.zig.Token.Id.AngleBracketLeftEqual,
- std.zig.Token.Id.AngleBracketAngleBracketLeft,
- std.zig.Token.Id.AngleBracketAngleBracketLeftEqual,
- std.zig.Token.Id.AngleBracketRight,
- std.zig.Token.Id.AngleBracketRightEqual,
- std.zig.Token.Id.AngleBracketAngleBracketRight,
- std.zig.Token.Id.AngleBracketAngleBracketRightEqual,
- std.zig.Token.Id.Tilde,
- std.zig.Token.Id.BracketStarBracket,
- std.zig.Token.Id.BracketStarCBracket,
+ .Bang,
+ .Pipe,
+ .PipePipe,
+ .PipeEqual,
+ .Equal,
+ .EqualEqual,
+ .EqualAngleBracketRight,
+ .BangEqual,
+ .LParen,
+ .RParen,
+ .Semicolon,
+ .Percent,
+ .PercentEqual,
+ .LBrace,
+ .RBrace,
+ .LBracket,
+ .RBracket,
+ .Period,
+ .Ellipsis2,
+ .Ellipsis3,
+ .Caret,
+ .CaretEqual,
+ .Plus,
+ .PlusPlus,
+ .PlusEqual,
+ .PlusPercent,
+ .PlusPercentEqual,
+ .Minus,
+ .MinusEqual,
+ .MinusPercent,
+ .MinusPercentEqual,
+ .Asterisk,
+ .AsteriskEqual,
+ .AsteriskAsterisk,
+ .AsteriskPercent,
+ .AsteriskPercentEqual,
+ .Arrow,
+ .Colon,
+ .Slash,
+ .SlashEqual,
+ .Comma,
+ .Ampersand,
+ .AmpersandEqual,
+ .QuestionMark,
+ .AngleBracketLeft,
+ .AngleBracketLeftEqual,
+ .AngleBracketAngleBracketLeft,
+ .AngleBracketAngleBracketLeftEqual,
+ .AngleBracketRight,
+ .AngleBracketRightEqual,
+ .AngleBracketAngleBracketRight,
+ .AngleBracketAngleBracketRightEqual,
+ .Tilde,
+ .BracketStarBracket,
+ .BracketStarCBracket,
=> try writeEscaped(out, src[token.start..token.end]),
- std.zig.Token.Id.AmpersandAmpersand,
- std.zig.Token.Id.Invalid,
- => return parseError(
+ .Invalid, .Invalid_ampersands => return parseError(
docgen_tokenizer,
source_token,
"syntax error",
diff --git a/std/zig/ast.zig b/std/zig/ast.zig
@@ -113,7 +113,6 @@ pub const Tree = struct {
pub const Error = union(enum) {
InvalidToken: InvalidToken,
- InvalidAmpersandAmpersand: InvalidAmpersandAmpersand,
ExpectedContainerMembers: ExpectedContainerMembers,
ExpectedStringLiteral: ExpectedStringLiteral,
ExpectedIntegerLiteral: ExpectedIntegerLiteral,
@@ -161,7 +160,6 @@ pub const Error = union(enum) {
pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
switch (self.*) {
.InvalidToken => |*x| return x.render(tokens, stream),
- .InvalidAmpersandAmpersand => |*x| return x.render(tokens, stream),
.ExpectedContainerMembers => |*x| return x.render(tokens, stream),
.ExpectedStringLiteral => |*x| return x.render(tokens, stream),
.ExpectedIntegerLiteral => |*x| return x.render(tokens, stream),
@@ -211,7 +209,6 @@ pub const Error = union(enum) {
pub fn loc(self: *const Error) TokenIndex {
switch (self.*) {
.InvalidToken => |x| return x.token,
- .InvalidAmpersandAmpersand => |x| return x.token,
.ExpectedContainerMembers => |x| return x.token,
.ExpectedStringLiteral => |x| return x.token,
.ExpectedIntegerLiteral => |x| return x.token,
@@ -292,7 +289,6 @@ pub const Error = union(enum) {
pub const ExpectedDerefOrUnwrap = SingleTokenError("Expected pointer dereference or optional unwrap, found {}");
pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found {}");
- pub const InvalidAmpersandAmpersand = SimpleError("Invalid token '&&', 'and' performs boolean AND");
pub const ExpectedParamType = SimpleError("Expected parameter type");
pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member");
pub const UnattachedDocComment = SimpleError("Unattached documentation comment");
@@ -322,8 +318,19 @@ pub const Error = union(enum) {
expected_id: Token.Id,
pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
- const token_name = @tagName(tokens.at(self.token).id);
- return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name);
+ const found_token = tokens.at(self.token);
+ switch (found_token.id) {
+ .Invalid_ampersands => {
+ return stream.print("`&&` is invalid. Note that `and` is boolean AND.");
+ },
+ .Invalid => {
+ return stream.print("expected {}, found invalid bytes", @tagName(self.expected_id));
+ },
+ else => {
+ const token_name = @tagName(found_token.id);
+ return stream.print("expected {}, found {}", @tagName(self.expected_id), token_name);
+ },
+ }
}
};
diff --git a/std/zig/parse.zig b/std/zig/parse.zig
@@ -774,7 +774,7 @@ fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
arena,
it,
tree,
- parseAmpersandAmpersandOp,
+ SimpleBinOpParseFn(.Keyword_and, .BoolAnd),
parseCompareExpr,
.Infinitely,
);
@@ -2687,18 +2687,6 @@ fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) No
// Helper parsers not included in the grammar
-fn parseAmpersandAmpersandOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
- const op_parse_and = SimpleBinOpParseFn(.Keyword_and, Node.InfixOp.Op.BoolAnd);
- const op_token = eatToken(it, .AmpersandAmpersand);
- if (op_token != null) {
- try tree.errors.push(AstError{
- .InvalidAmpersandAmpersand = AstError.InvalidAmpersandAmpersand{ .token = it.index },
- });
- return error.ParseError;
- }
- return op_parse_and(arena, it, tree);
-}
-
fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
const token = eatToken(it, .Builtin) orelse return null;
const params = (try parseFnCallArguments(arena, it, tree)) orelse {
diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig
@@ -77,6 +77,7 @@ pub const Token = struct {
pub const Id = enum {
Invalid,
+ Invalid_ampersands,
Identifier,
StringLiteral,
MultilineStringLiteralLine,
@@ -125,7 +126,6 @@ pub const Token = struct {
SlashEqual,
Comma,
Ampersand,
- AmpersandAmpersand,
AmpersandEqual,
QuestionMark,
AngleBracketLeft,
@@ -486,7 +486,8 @@ pub const Tokenizer = struct {
State.Ampersand => switch (c) {
'&' => {
- result.id = Token.Id.AmpersandAmpersand;
+ result.id = Token.Id.Invalid_ampersands;
+ self.index += 1;
break;
},
'=' => {
