motiejus/zig0
1
Fork 0
Code Releases Activity

replace TOKENIZER_TAG_ with TOKEN_

Browse Source
This commit is contained in:
Motiejus Jakštys
2024-12-30 22:38:17 +02:00
parent d551ba3d12
commit 3264d1747e
5 changed files with 511 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ast.c
View File

@@ -31,7 +31,7 @@ Ast astParse(const char* source, const uint32_t len) {
tokens.tags[tokens.len] = token.tag;
tokens.starts[tokens.len] = token.loc.start;
tokens.len++;
if (token.tag == TOKENIZER_TAG_EOF)
if (token.tag == TOKEN_EOF)
break;
}

244
parser.c
View File

@@ -74,12 +74,12 @@ static AstTokenIndex eatToken(Parser* p, TokenizerTag tag, bool* ok) {
static void eatDocComments(Parser* p) {
bool ok;
while (eatToken(p, TOKENIZER_TAG_DOC_COMMENT, &ok), ok) { }
while (eatToken(p, TOKEN_DOC_COMMENT, &ok), ok) { }
}
static void expectSemicolon(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_SEMICOLON, &ok);
eatToken(p, TOKEN_SEMICOLON, &ok);
if (ok)
return;
@@ -126,7 +126,7 @@ static AstNodeIndex addExtra(Parser* p, const AstNodeIndex* extra, uint32_t coun
static AstNodeIndex parseByteAlign(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_KEYWORD_ALIGN, &ok);
eatToken(p, TOKEN_KEYWORD_ALIGN, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseByteAlign cannot parse alginment\n");
@@ -136,7 +136,7 @@ static AstNodeIndex parseByteAlign(Parser* p) {
static AstNodeIndex parseAddrSpace(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_KEYWORD_ADDRSPACE, &ok);
eatToken(p, TOKEN_KEYWORD_ADDRSPACE, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseAddrSpace cannot parse addrspace\n");
@@ -146,7 +146,7 @@ static AstNodeIndex parseAddrSpace(Parser* p) {
static AstNodeIndex parseLinkSection(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_KEYWORD_LINKSECTION, &ok);
eatToken(p, TOKEN_KEYWORD_LINKSECTION, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseLinkSection cannot parse linksection\n");
@@ -156,7 +156,7 @@ static AstNodeIndex parseLinkSection(Parser* p) {
static AstNodeIndex parseCallconv(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_KEYWORD_CALLCONV, &ok);
eatToken(p, TOKEN_KEYWORD_CALLCONV, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseCallconv cannot parse callconv\n");
@@ -171,15 +171,15 @@ typedef struct {
} NodeContainerField;
static AstNodeIndex expectContainerField(Parser* p) {
eatToken(p, TOKENIZER_TAG_KEYWORD_COMPTIME, NULL);
eatToken(p, TOKEN_KEYWORD_COMPTIME, NULL);
const AstTokenIndex main_token = p->tok_i;
if (p->token_tags[p->tok_i] == TOKENIZER_TAG_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKENIZER_TAG_COLON)
if (p->token_tags[p->tok_i] == TOKEN_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKEN_COLON)
p->tok_i += 2;
const AstNodeIndex type_expr = parseTypeExpr(p);
const AstNodeIndex align_expr = parseByteAlign(p);
bool ok;
eatToken(p, TOKENIZER_TAG_EQUAL, &ok);
eatToken(p, TOKEN_EQUAL, &ok);
if (ok) {
fprintf(stderr, "expectContainerField does not support expr\n");
exit(1);
@@ -225,27 +225,27 @@ static AstNodeIndex expectContainerField(Parser* p) {
static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
const TokenizerTag tok = p->token_tags[p->tok_i];
switch (tok) {
case TOKENIZER_TAG_CHAR_LITERAL:
case TOKENIZER_TAG_NUMBER_LITERAL:
case TOKENIZER_TAG_KEYWORD_UNREACHABLE:
case TOKENIZER_TAG_KEYWORD_ANYFRAME:
case TOKENIZER_TAG_STRING_LITERAL:
case TOKENIZER_TAG_BUILTIN:
case TOKENIZER_TAG_KEYWORD_FN:
case TOKENIZER_TAG_KEYWORD_IF:
case TOKENIZER_TAG_KEYWORD_SWITCH:
case TOKENIZER_TAG_KEYWORD_EXTERN:
case TOKENIZER_TAG_KEYWORD_PACKED:
case TOKENIZER_TAG_KEYWORD_STRUCT:
case TOKENIZER_TAG_KEYWORD_OPAQUE:
case TOKENIZER_TAG_KEYWORD_ENUM:
case TOKENIZER_TAG_KEYWORD_UNION:
case TOKENIZER_TAG_KEYWORD_COMPTIME:
case TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE:
case TOKEN_CHAR_LITERAL:
case TOKEN_NUMBER_LITERAL:
case TOKEN_KEYWORD_UNREACHABLE:
case TOKEN_KEYWORD_ANYFRAME:
case TOKEN_STRING_LITERAL:
case TOKEN_BUILTIN:
case TOKEN_KEYWORD_FN:
case TOKEN_KEYWORD_IF:
case TOKEN_KEYWORD_SWITCH:
case TOKEN_KEYWORD_EXTERN:
case TOKEN_KEYWORD_PACKED:
case TOKEN_KEYWORD_STRUCT:
case TOKEN_KEYWORD_OPAQUE:
case TOKEN_KEYWORD_ENUM:
case TOKEN_KEYWORD_UNION:
case TOKEN_KEYWORD_COMPTIME:
case TOKEN_MULTILINE_STRING_LITERAL_LINE:
fprintf(stderr, "parsePrimaryTypeExpr does not support %s\n", tokenizerGetTagString(tok));
exit(1);
case TOKENIZER_TAG_IDENTIFIER:
if (p->token_tags[p->tok_i + 1] == TOKENIZER_TAG_COLON) {
case TOKEN_IDENTIFIER:
if (p->token_tags[p->tok_i + 1] == TOKEN_COLON) {
fprintf(stderr, "parsePrimaryTypeExpr does not support identifier followed by colon\n");
exit(1);
}
@@ -255,12 +255,12 @@ static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
.tag = AST_NODE_TAG_IDENTIFIER,
.main_token = nextToken(p),
.data = {} });
case TOKENIZER_TAG_KEYWORD_INLINE:
case TOKENIZER_TAG_KEYWORD_FOR:
case TOKENIZER_TAG_KEYWORD_WHILE:
case TOKENIZER_TAG_PERIOD:
case TOKENIZER_TAG_KEYWORD_ERROR:
case TOKENIZER_TAG_L_PAREN:
case TOKEN_KEYWORD_INLINE:
case TOKEN_KEYWORD_FOR:
case TOKEN_KEYWORD_WHILE:
case TOKEN_PERIOD:
case TOKEN_KEYWORD_ERROR:
case TOKEN_L_PAREN:
fprintf(stderr, "parsePrimaryTypeExpr does not support %s\n", tokenizerGetTagString(tok));
exit(1);
default:
@@ -271,10 +271,10 @@ static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
static AstNodeIndex parseSuffixOp(Parser* p) {
const TokenizerTag tok = p->token_tags[p->tok_i];
switch (tok) {
case TOKENIZER_TAG_L_BRACKET:
case TOKENIZER_TAG_PERIOD_ASTERISK:
case TOKENIZER_TAG_INVALID_PERIODASTERISKS:
case TOKENIZER_TAG_PERIOD:
case TOKEN_L_BRACKET:
case TOKEN_PERIOD_ASTERISK:
case TOKEN_INVALID_PERIODASTERISKS:
case TOKEN_PERIOD:
fprintf(stderr, "parseSuffixOp does not support %s\n", tokenizerGetTagString(tok));
exit(1);
default:
@@ -284,7 +284,7 @@ static AstNodeIndex parseSuffixOp(Parser* p) {
static AstNodeIndex parseSuffixExpr(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_KEYWORD_ASYNC, &ok);
eatToken(p, TOKEN_KEYWORD_ASYNC, &ok);
if (ok) {
fprintf(stderr, "async not supported\n");
exit(1);
@@ -300,13 +300,13 @@ static AstNodeIndex parseSuffixExpr(Parser* p) {
res = suffix_op;
continue;
}
eatToken(p, TOKENIZER_TAG_L_PAREN, &ok);
eatToken(p, TOKEN_L_PAREN, &ok);
if (ok) {
fprintf(stderr, "parseSuffixExpr does not support expr with parens\n");
exit(1);
}
// TODO more work
// const bool comma = p->token_tags[p->tok_i - 2] == TOKENIZER_TAG_COMMA;
// const bool comma = p->token_tags[p->tok_i - 2] == TOKEN_COMMA;
return res;
}
@@ -329,7 +329,7 @@ static AstNodeIndex parseErrorUnionExpr(Parser* p) {
if (suffix_expr == 0)
return null_node;
bool ok;
const AstNodeIndex bang = eatToken(p, TOKENIZER_TAG_BANG, &ok);
const AstNodeIndex bang = eatToken(p, TOKEN_BANG, &ok);
if (!ok)
return suffix_expr;
return addNode(
@@ -347,11 +347,11 @@ static AstNodeIndex parseErrorUnionExpr(Parser* p) {
static AstNodeIndex parseTypeExpr(Parser* p) {
const AstNodeIndex tok = p->token_tags[p->tok_i];
switch (tok) {
case TOKENIZER_TAG_QUESTION_MARK:
case TOKENIZER_TAG_KEYWORD_ANYFRAME:
case TOKENIZER_TAG_ASTERISK:
case TOKENIZER_TAG_ASTERISK_ASTERISK:
case TOKENIZER_TAG_L_BRACKET:
case TOKEN_QUESTION_MARK:
case TOKEN_KEYWORD_ANYFRAME:
case TOKEN_ASTERISK:
case TOKEN_ASTERISK_ASTERISK:
case TOKEN_L_BRACKET:
fprintf(stderr, "parseTypeExpr not supported for %s\n", tokenizerGetTagString(tok));
exit(1);
default:
@@ -362,13 +362,13 @@ static AstNodeIndex parseTypeExpr(Parser* p) {
static SmallSpan parseParamDeclList(Parser* p) {
// can only parse functions with no declarations
bool ok;
AstTokenIndex got_token = eatToken(p, TOKENIZER_TAG_L_PAREN, &ok);
AstTokenIndex got_token = eatToken(p, TOKEN_L_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected (, got %s\n", tokenizerGetTagString(got_token));
exit(1);
}
got_token = eatToken(p, TOKENIZER_TAG_R_PAREN, &ok);
got_token = eatToken(p, TOKEN_R_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected ), got %s\n", tokenizerGetTagString(got_token));
exit(1);
@@ -387,20 +387,20 @@ static uint32_t reserveNode(Parser* p, AstNodeTag tag) {
static AstNodeIndex parseFnProto(Parser* p) {
bool ok;
AstNodeIndex fn_token = eatToken(p, TOKENIZER_TAG_KEYWORD_FN, &ok);
AstNodeIndex fn_token = eatToken(p, TOKEN_KEYWORD_FN, &ok);
if (!ok)
return null_node;
AstNodeIndex fn_proto_index = reserveNode(p, AST_NODE_TAG_FN_PROTO);
eatToken(p, TOKENIZER_TAG_IDENTIFIER, NULL);
eatToken(p, TOKEN_IDENTIFIER, NULL);
SmallSpan params = parseParamDeclList(p);
const AstNodeIndex align_expr = parseByteAlign(p);
const AstNodeIndex addrspace_expr = parseAddrSpace(p);
const AstNodeIndex section_expr = parseLinkSection(p);
const AstNodeIndex callconv_expr = parseCallconv(p);
eatToken(p, TOKENIZER_TAG_BANG, NULL);
eatToken(p, TOKEN_BANG, NULL);
const AstNodeIndex return_type_expr = parseTypeExpr(p);
@@ -425,7 +425,7 @@ static AstNodeIndex parseFnProto(Parser* p) {
}
static AstTokenIndex parseBlockLabel(Parser* p) {
if (p->token_tags[p->tok_i] == TOKENIZER_TAG_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKENIZER_TAG_COLON) {
if (p->token_tags[p->tok_i] == TOKEN_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKEN_COLON) {
const AstTokenIndex identifier = p->tok_i;
p->tok_i += 2;
return identifier;
@@ -435,7 +435,7 @@ static AstTokenIndex parseBlockLabel(Parser* p) {
static AstNodeIndex parseForStatement(Parser* p) {
bool ok;
const AstNodeIndex for_token = eatToken(p, TOKENIZER_TAG_KEYWORD_FOR, &ok);
const AstNodeIndex for_token = eatToken(p, TOKEN_KEYWORD_FOR, &ok);
if (!ok)
return null_node;
@@ -446,7 +446,7 @@ static AstNodeIndex parseForStatement(Parser* p) {
static AstNodeIndex parseWhileStatement(Parser* p) {
bool ok;
const AstNodeIndex while_token = eatToken(p, TOKENIZER_TAG_KEYWORD_WHILE, &ok);
const AstNodeIndex while_token = eatToken(p, TOKEN_KEYWORD_WHILE, &ok);
if (!ok)
return null_node;
@@ -457,7 +457,7 @@ static AstNodeIndex parseWhileStatement(Parser* p) {
static AstNodeIndex parseLoopStatement(Parser* p) {
bool ok_inline_token;
eatToken(p, TOKENIZER_TAG_KEYWORD_INLINE, &ok_inline_token);
eatToken(p, TOKEN_KEYWORD_INLINE, &ok_inline_token);
const AstNodeIndex for_statement = parseForStatement(p);
if (for_statement != 0)
@@ -492,21 +492,21 @@ static AstNodeIndex expectVarDeclExprStatement(Parser* p) {
static AstNodeIndex parseLabeledStatement(Parser*);
static AstNodeIndex expectStatement(Parser* p, bool allow_defer_var) {
bool ok;
if (eatToken(p, TOKENIZER_TAG_KEYWORD_COMPTIME, &ok), ok) {
if (eatToken(p, TOKEN_KEYWORD_COMPTIME, &ok), ok) {
fprintf(stderr, "expectStatement: comptime keyword not supported\n");
exit(1);
}
const AstNodeIndex tok = p->token_tags[p->tok_i];
switch (tok) {
case TOKENIZER_TAG_KEYWORD_NOSUSPEND:
case TOKENIZER_TAG_KEYWORD_SUSPEND:
case TOKENIZER_TAG_KEYWORD_DEFER:
case TOKENIZER_TAG_KEYWORD_ERRDEFER:
case TOKENIZER_TAG_KEYWORD_IF:
case TOKENIZER_TAG_KEYWORD_ENUM:
case TOKENIZER_TAG_KEYWORD_STRUCT:
case TOKENIZER_TAG_KEYWORD_UNION:;
case TOKEN_KEYWORD_NOSUSPEND:
case TOKEN_KEYWORD_SUSPEND:
case TOKEN_KEYWORD_DEFER:
case TOKEN_KEYWORD_ERRDEFER:
case TOKEN_KEYWORD_IF:
case TOKEN_KEYWORD_ENUM:
case TOKEN_KEYWORD_STRUCT:
case TOKEN_KEYWORD_UNION:;
const char* tok_str = tokenizerGetTagString(tok);
fprintf(stderr, "expectStatement does not support keyword %s\n", tok_str);
exit(1);
@@ -532,7 +532,7 @@ static void cleanupScratch(CleanupScratch* c) { c->scratch->len = c->old_len; }
static AstNodeIndex parseBlock(Parser* p) {
bool ok;
const AstNodeIndex lbrace = eatToken(p, TOKENIZER_TAG_L_BRACE, &ok);
const AstNodeIndex lbrace = eatToken(p, TOKEN_L_BRACE, &ok);
if (!ok)
return null_node;
@@ -542,7 +542,7 @@ static AstNodeIndex parseBlock(Parser* p) {
};
while (1) {
if (p->token_tags[p->tok_i] == TOKENIZER_TAG_R_BRACE)
if (p->token_tags[p->tok_i] == TOKEN_R_BRACE)
break;
// "const AstNodeIndex statement" once tinycc supports typeof_unqual (C23)
@@ -551,8 +551,8 @@ static AstNodeIndex parseBlock(Parser* p) {
break;
SLICE_APPEND(AstNodeIndex, &p->scratch, statement);
}
expectToken(p, TOKENIZER_TAG_R_BRACE, NULL);
const bool semicolon = (p->token_tags[p->tok_i] - 2 == TOKENIZER_TAG_SEMICOLON);
expectToken(p, TOKEN_R_BRACE, NULL);
const bool semicolon = (p->token_tags[p->tok_i] - 2 == TOKEN_SEMICOLON);
const uint32_t statements_len = p->scratch.len - scratch_top.old_len;
switch (statements_len) {
@@ -626,9 +626,9 @@ static AstNodeIndex parseLabeledStatement(Parser* p) {
static AstNodeIndex parseVarDeclProto(Parser* p) {
bool ok;
eatToken(p, TOKENIZER_TAG_KEYWORD_CONST, &ok);
eatToken(p, TOKEN_KEYWORD_CONST, &ok);
if (!ok) {
eatToken(p, TOKENIZER_TAG_KEYWORD_VAR, &ok);
eatToken(p, TOKEN_KEYWORD_VAR, &ok);
if (!ok)
return null_node;
}
@@ -651,12 +651,12 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
AstTokenIndex extern_export_inline_token = nextToken(p);
switch (p->token_tags[extern_export_inline_token]) {
case TOKENIZER_TAG_KEYWORD_EXTERN:
eatToken(p, TOKENIZER_TAG_STRING_LITERAL, NULL);
case TOKEN_KEYWORD_EXTERN:
eatToken(p, TOKEN_STRING_LITERAL, NULL);
break;
case TOKENIZER_TAG_KEYWORD_EXPORT:
case TOKENIZER_TAG_KEYWORD_INLINE:
case TOKENIZER_TAG_KEYWORD_NOINLINE:
case TOKEN_KEYWORD_EXPORT:
case TOKEN_KEYWORD_INLINE:
case TOKEN_KEYWORD_NOINLINE:
break;
default:
p->tok_i--;
@@ -665,10 +665,10 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
AstNodeIndex fn_proto = parseFnProto(p);
if (fn_proto != 0) {
switch (p->token_tags[p->tok_i]) {
case TOKENIZER_TAG_SEMICOLON:
case TOKEN_SEMICOLON:
p->tok_i++;
return fn_proto;
case TOKENIZER_TAG_L_BRACE:;
case TOKEN_L_BRACE:;
AstNodeIndex fn_decl_index = reserveNode(p, AST_NODE_TAG_FN_DECL);
AstNodeIndex body_block = parseBlock(p);
return setNode(
@@ -684,7 +684,7 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
}
}
eatToken(p, TOKENIZER_TAG_KEYWORD_THREADLOCAL, NULL);
eatToken(p, TOKEN_KEYWORD_THREADLOCAL, NULL);
AstNodeIndex var_decl = parseGlobalVarDecl(p);
if (var_decl != 0) {
return var_decl;
@@ -704,46 +704,46 @@ void findNextContainerMember(Parser* p) {
switch (p->token_tags[tok]) {
// Any of these can start a new top level declaration
case TOKENIZER_TAG_KEYWORD_TEST:
case TOKENIZER_TAG_KEYWORD_COMPTIME:
case TOKENIZER_TAG_KEYWORD_PUB:
case TOKENIZER_TAG_KEYWORD_EXPORT:
case TOKENIZER_TAG_KEYWORD_EXTERN:
case TOKENIZER_TAG_KEYWORD_INLINE:
case TOKENIZER_TAG_KEYWORD_NOINLINE:
case TOKENIZER_TAG_KEYWORD_USINGNAMESPACE:
case TOKENIZER_TAG_KEYWORD_THREADLOCAL:
case TOKENIZER_TAG_KEYWORD_CONST:
case TOKENIZER_TAG_KEYWORD_VAR:
case TOKENIZER_TAG_KEYWORD_FN:
case TOKEN_KEYWORD_TEST:
case TOKEN_KEYWORD_COMPTIME:
case TOKEN_KEYWORD_PUB:
case TOKEN_KEYWORD_EXPORT:
case TOKEN_KEYWORD_EXTERN:
case TOKEN_KEYWORD_INLINE:
case TOKEN_KEYWORD_NOINLINE:
case TOKEN_KEYWORD_USINGNAMESPACE:
case TOKEN_KEYWORD_THREADLOCAL:
case TOKEN_KEYWORD_CONST:
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_FN:
if (level == 0) {
p->tok_i--;
return;
}
break;
case TOKENIZER_TAG_IDENTIFIER:
if (p->token_tags[tok + 1] == TOKENIZER_TAG_COMMA && level == 0) {
case TOKEN_IDENTIFIER:
if (p->token_tags[tok + 1] == TOKEN_COMMA && level == 0) {
p->tok_i--;
return;
}
break;
case TOKENIZER_TAG_COMMA:
case TOKENIZER_TAG_SEMICOLON:
case TOKEN_COMMA:
case TOKEN_SEMICOLON:
// This decl was likely meant to end here
if (level == 0)
return;
break;
case TOKENIZER_TAG_L_PAREN:
case TOKENIZER_TAG_L_BRACKET:
case TOKENIZER_TAG_L_BRACE:
case TOKEN_L_PAREN:
case TOKEN_L_BRACKET:
case TOKEN_L_BRACE:
level++;
break;
case TOKENIZER_TAG_R_PAREN:
case TOKENIZER_TAG_R_BRACKET:
case TOKEN_R_PAREN:
case TOKEN_R_BRACKET:
if (level != 0)
level--;
break;
case TOKENIZER_TAG_R_BRACE:
case TOKEN_R_BRACE:
if (level == 0) {
// end of container, exit
p->tok_i--;
@@ -751,7 +751,7 @@ void findNextContainerMember(Parser* p) {
}
level--;
break;
case TOKENIZER_TAG_EOF:
case TOKEN_EOF:
p->tok_i--;
return;
default:
@@ -766,7 +766,7 @@ static Members parseContainerMembers(Parser* p) {
.old_len = p->scratch.len,
};
bool ok;
while (eatToken(p, TOKENIZER_TAG_CONTAINER_DOC_COMMENT, &ok), ok)
while (eatToken(p, TOKEN_CONTAINER_DOC_COMMENT, &ok), ok)
;
FieldState field_state = { .tag = FIELD_STATE_NONE };
@@ -775,13 +775,13 @@ static Members parseContainerMembers(Parser* p) {
while (1) {
eatDocComments(p);
switch (p->token_tags[p->tok_i]) {
case TOKENIZER_TAG_KEYWORD_TEST:
case TOKENIZER_TAG_KEYWORD_COMPTIME:
case TOKENIZER_TAG_KEYWORD_USINGNAMESPACE:;
case TOKEN_KEYWORD_TEST:
case TOKEN_KEYWORD_COMPTIME:
case TOKEN_KEYWORD_USINGNAMESPACE:;
const char* str = tokenizerGetTagString(p->token_tags[p->tok_i]);
fprintf(stderr, "%s not implemented in parseContainerMembers\n", str);
exit(1);
case TOKENIZER_TAG_KEYWORD_PUB: {
case TOKEN_KEYWORD_PUB: {
p->tok_i++;
AstNodeIndex top_level_decl = expectTopLevelDecl(p);
if (top_level_decl != 0) {
@@ -791,17 +791,17 @@ static Members parseContainerMembers(Parser* p) {
}
SLICE_APPEND(AstNodeIndex, &p->scratch, top_level_decl);
}
trailing = p->token_tags[p->tok_i - 1] == TOKENIZER_TAG_SEMICOLON;
trailing = p->token_tags[p->tok_i - 1] == TOKEN_SEMICOLON;
break;
}
case TOKENIZER_TAG_KEYWORD_CONST:
case TOKENIZER_TAG_KEYWORD_VAR:
case TOKENIZER_TAG_KEYWORD_THREADLOCAL:
case TOKENIZER_TAG_KEYWORD_EXPORT:
case TOKENIZER_TAG_KEYWORD_EXTERN:
case TOKENIZER_TAG_KEYWORD_INLINE:
case TOKENIZER_TAG_KEYWORD_NOINLINE:
case TOKENIZER_TAG_KEYWORD_FN: {
case TOKEN_KEYWORD_CONST:
case TOKEN_KEYWORD_VAR:
case TOKEN_KEYWORD_THREADLOCAL:
case TOKEN_KEYWORD_EXPORT:
case TOKEN_KEYWORD_EXTERN:
case TOKEN_KEYWORD_INLINE:
case TOKEN_KEYWORD_NOINLINE:
case TOKEN_KEYWORD_FN: {
const AstNodeIndex top_level_decl = expectTopLevelDecl(p);
if (top_level_decl != 0) {
if (field_state.tag == FIELD_STATE_SEEN) {
@@ -810,11 +810,11 @@ static Members parseContainerMembers(Parser* p) {
}
SLICE_APPEND(AstNodeIndex, &p->scratch, top_level_decl);
}
trailing = (p->token_tags[p->tok_i - 1] == TOKENIZER_TAG_SEMICOLON);
trailing = (p->token_tags[p->tok_i - 1] == TOKEN_SEMICOLON);
break;
}
case TOKENIZER_TAG_EOF:
case TOKENIZER_TAG_R_BRACE:
case TOKEN_EOF:
case TOKEN_R_BRACE:
goto break_loop;
default:;
// skip parseCStyleContainer
@@ -831,12 +831,12 @@ static Members parseContainerMembers(Parser* p) {
}
SLICE_APPEND(AstNodeIndex, &p->scratch, container_field);
switch (p->token_tags[p->tok_i]) {
case TOKENIZER_TAG_COMMA:
case TOKEN_COMMA:
p->tok_i++;
trailing = true;
continue;
case TOKENIZER_TAG_R_BRACE:
case TOKENIZER_TAG_EOF:
case TOKEN_R_BRACE:
case TOKEN_EOF:
trailing = false;
goto break_loop;
default:;

288
tokenizer.c
View File

@@ -19,55 +19,55 @@ const char* tokenizerGetTagString(TokenizerTag tag) {
}
const KeywordMap keywords[] = {
{ "addrspace", TOKENIZER_TAG_KEYWORD_ADDRSPACE },
{ "align", TOKENIZER_TAG_KEYWORD_ALIGN },
{ "allowzero", TOKENIZER_TAG_KEYWORD_ALLOWZERO },
{ "and", TOKENIZER_TAG_KEYWORD_AND },
{ "anyframe", TOKENIZER_TAG_KEYWORD_ANYFRAME },
{ "anytype", TOKENIZER_TAG_KEYWORD_ANYTYPE },
{ "asm", TOKENIZER_TAG_KEYWORD_ASM },
{ "async", TOKENIZER_TAG_KEYWORD_ASYNC },
{ "await", TOKENIZER_TAG_KEYWORD_AWAIT },
{ "break", TOKENIZER_TAG_KEYWORD_BREAK },
{ "callconv", TOKENIZER_TAG_KEYWORD_CALLCONV },
{ "catch", TOKENIZER_TAG_KEYWORD_CATCH },
{ "comptime", TOKENIZER_TAG_KEYWORD_COMPTIME },
{ "const", TOKENIZER_TAG_KEYWORD_CONST },
{ "continue", TOKENIZER_TAG_KEYWORD_CONTINUE },
{ "defer", TOKENIZER_TAG_KEYWORD_DEFER },
{ "else", TOKENIZER_TAG_KEYWORD_ELSE },
{ "enum", TOKENIZER_TAG_KEYWORD_ENUM },
{ "errdefer", TOKENIZER_TAG_KEYWORD_ERRDEFER },
{ "error", TOKENIZER_TAG_KEYWORD_ERROR },
{ "export", TOKENIZER_TAG_KEYWORD_EXPORT },
{ "extern", TOKENIZER_TAG_KEYWORD_EXTERN },
{ "fn", TOKENIZER_TAG_KEYWORD_FN },
{ "for", TOKENIZER_TAG_KEYWORD_FOR },
{ "if", TOKENIZER_TAG_KEYWORD_IF },
{ "inline", TOKENIZER_TAG_KEYWORD_INLINE },
{ "linksection", TOKENIZER_TAG_KEYWORD_LINKSECTION },
{ "noalias", TOKENIZER_TAG_KEYWORD_NOALIAS },
{ "noinline", TOKENIZER_TAG_KEYWORD_NOINLINE },
{ "nosuspend", TOKENIZER_TAG_KEYWORD_NOSUSPEND },
{ "opaque", TOKENIZER_TAG_KEYWORD_OPAQUE },
{ "or", TOKENIZER_TAG_KEYWORD_OR },
{ "orelse", TOKENIZER_TAG_KEYWORD_ORELSE },
{ "packed", TOKENIZER_TAG_KEYWORD_PACKED },
{ "pub", TOKENIZER_TAG_KEYWORD_PUB },
{ "resume", TOKENIZER_TAG_KEYWORD_RESUME },
{ "return", TOKENIZER_TAG_KEYWORD_RETURN },
{ "struct", TOKENIZER_TAG_KEYWORD_STRUCT },
{ "suspend", TOKENIZER_TAG_KEYWORD_SUSPEND },
{ "switch", TOKENIZER_TAG_KEYWORD_SWITCH },
{ "test", TOKENIZER_TAG_KEYWORD_TEST },
{ "threadlocal", TOKENIZER_TAG_KEYWORD_THREADLOCAL },
{ "try", TOKENIZER_TAG_KEYWORD_TRY },
{ "union", TOKENIZER_TAG_KEYWORD_UNION },
{ "unreachable", TOKENIZER_TAG_KEYWORD_UNREACHABLE },
{ "usingnamespace", TOKENIZER_TAG_KEYWORD_USINGNAMESPACE },
{ "var", TOKENIZER_TAG_KEYWORD_VAR },
{ "volatile", TOKENIZER_TAG_KEYWORD_VOLATILE },
{ "while", TOKENIZER_TAG_KEYWORD_WHILE }
{ "addrspace", TOKEN_KEYWORD_ADDRSPACE },
{ "align", TOKEN_KEYWORD_ALIGN },
{ "allowzero", TOKEN_KEYWORD_ALLOWZERO },
{ "and", TOKEN_KEYWORD_AND },
{ "anyframe", TOKEN_KEYWORD_ANYFRAME },
{ "anytype", TOKEN_KEYWORD_ANYTYPE },
{ "asm", TOKEN_KEYWORD_ASM },
{ "async", TOKEN_KEYWORD_ASYNC },
{ "await", TOKEN_KEYWORD_AWAIT },
{ "break", TOKEN_KEYWORD_BREAK },
{ "callconv", TOKEN_KEYWORD_CALLCONV },
{ "catch", TOKEN_KEYWORD_CATCH },
{ "comptime", TOKEN_KEYWORD_COMPTIME },
{ "const", TOKEN_KEYWORD_CONST },
{ "continue", TOKEN_KEYWORD_CONTINUE },
{ "defer", TOKEN_KEYWORD_DEFER },
{ "else", TOKEN_KEYWORD_ELSE },
{ "enum", TOKEN_KEYWORD_ENUM },
{ "errdefer", TOKEN_KEYWORD_ERRDEFER },
{ "error", TOKEN_KEYWORD_ERROR },
{ "export", TOKEN_KEYWORD_EXPORT },
{ "extern", TOKEN_KEYWORD_EXTERN },
{ "fn", TOKEN_KEYWORD_FN },
{ "for", TOKEN_KEYWORD_FOR },
{ "if", TOKEN_KEYWORD_IF },
{ "inline", TOKEN_KEYWORD_INLINE },
{ "linksection", TOKEN_KEYWORD_LINKSECTION },
{ "noalias", TOKEN_KEYWORD_NOALIAS },
{ "noinline", TOKEN_KEYWORD_NOINLINE },
{ "nosuspend", TOKEN_KEYWORD_NOSUSPEND },
{ "opaque", TOKEN_KEYWORD_OPAQUE },
{ "or", TOKEN_KEYWORD_OR },
{ "orelse", TOKEN_KEYWORD_ORELSE },
{ "packed", TOKEN_KEYWORD_PACKED },
{ "pub", TOKEN_KEYWORD_PUB },
{ "resume", TOKEN_KEYWORD_RESUME },
{ "return", TOKEN_KEYWORD_RETURN },
{ "struct", TOKEN_KEYWORD_STRUCT },
{ "suspend", TOKEN_KEYWORD_SUSPEND },
{ "switch", TOKEN_KEYWORD_SWITCH },
{ "test", TOKEN_KEYWORD_TEST },
{ "threadlocal", TOKEN_KEYWORD_THREADLOCAL },
{ "try", TOKEN_KEYWORD_TRY },
{ "union", TOKEN_KEYWORD_UNION },
{ "unreachable", TOKEN_KEYWORD_UNREACHABLE },
{ "usingnamespace", TOKEN_KEYWORD_USINGNAMESPACE },
{ "var", TOKEN_KEYWORD_VAR },
{ "volatile", TOKEN_KEYWORD_VOLATILE },
{ "while", TOKEN_KEYWORD_WHILE }
};
// TODO binary search
@@ -80,13 +80,13 @@ static TokenizerTag getKeyword(const char* bytes, const uint32_t len) {
if (len == klen) {
return keywords[i].tag;
} else {
return TOKENIZER_TAG_INVALID;
return TOKEN_INVALID;
}
} else if (cmp < 0) {
return TOKENIZER_TAG_INVALID;
return TOKEN_INVALID;
}
}
return TOKENIZER_TAG_INVALID;
return TOKEN_INVALID;
}
Tokenizer tokenizerInit(const char* buffer, const uint32_t len) {
@@ -99,7 +99,7 @@ Tokenizer tokenizerInit(const char* buffer, const uint32_t len) {
TokenizerToken tokenizerNext(Tokenizer* self) {
TokenizerToken result = (TokenizerToken) {
.tag = TOKENIZER_TAG_INVALID,
.tag = TOKEN_INVALID,
.loc = {
.start = 0,
},
@@ -114,7 +114,7 @@ state:
case 0:
if (self->index == self->buffer_len) {
return (TokenizerToken) {
.tag = TOKENIZER_TAG_EOF,
.tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index,
@@ -132,17 +132,17 @@ state:
result.loc.start = self->index;
goto state;
case '"':
result.tag = TOKENIZER_TAG_STRING_LITERAL;
result.tag = TOKEN_STRING_LITERAL;
state = TOKENIZER_STATE_STRING_LITERAL;
goto state;
case '\'':
result.tag = TOKENIZER_TAG_CHAR_LITERAL;
result.tag = TOKEN_CHAR_LITERAL;
state = TOKENIZER_STATE_CHAR_LITERAL;
goto state;
case 'a' ... 'z':
case 'A' ... 'Z':
case '_':
result.tag = TOKENIZER_TAG_IDENTIFIER;
result.tag = TOKEN_IDENTIFIER;
state = TOKENIZER_STATE_IDENTIFIER;
goto state;
case '@':
@@ -158,35 +158,35 @@ state:
state = TOKENIZER_STATE_PIPE;
goto state;
case '(':
result.tag = TOKENIZER_TAG_L_PAREN;
result.tag = TOKEN_L_PAREN;
self->index++;
break;
case ')':
result.tag = TOKENIZER_TAG_R_PAREN;
result.tag = TOKEN_R_PAREN;
self->index++;
break;
case '[':
result.tag = TOKENIZER_TAG_L_BRACKET;
result.tag = TOKEN_L_BRACKET;
self->index++;
break;
case ']':
result.tag = TOKENIZER_TAG_R_BRACKET;
result.tag = TOKEN_R_BRACKET;
self->index++;
break;
case ';':
result.tag = TOKENIZER_TAG_SEMICOLON;
result.tag = TOKEN_SEMICOLON;
self->index++;
break;
case ',':
result.tag = TOKENIZER_TAG_COMMA;
result.tag = TOKEN_COMMA;
self->index++;
break;
case '?':
result.tag = TOKENIZER_TAG_QUESTION_MARK;
result.tag = TOKEN_QUESTION_MARK;
self->index++;
break;
case ':':
result.tag = TOKENIZER_TAG_COLON;
result.tag = TOKEN_COLON;
self->index++;
break;
case '%':
@@ -208,19 +208,19 @@ state:
state = TOKENIZER_STATE_CARET;
goto state;
case '\\':
result.tag = TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE;
result.tag = TOKEN_MULTILINE_STRING_LITERAL_LINE;
state = TOKENIZER_STATE_BACKSLASH;
goto state;
case '{':
result.tag = TOKENIZER_TAG_L_BRACE;
result.tag = TOKEN_L_BRACE;
self->index++;
break;
case '}':
result.tag = TOKENIZER_TAG_R_BRACE;
result.tag = TOKEN_R_BRACE;
self->index++;
break;
case '~':
result.tag = TOKENIZER_TAG_TILDE;
result.tag = TOKEN_TILDE;
self->index++;
break;
case '.':
@@ -236,7 +236,7 @@ state:
state = TOKENIZER_STATE_AMPERSAND;
goto state;
case '0' ... '9':
result.tag = TOKENIZER_TAG_NUMBER_LITERAL;
result.tag = TOKEN_NUMBER_LITERAL;
self->index++;
state = TOKENIZER_STATE_INT;
goto state;
@@ -251,7 +251,7 @@ state:
switch (self->buffer[self->index]) {
case 0:
if (self->index == self->buffer_len) {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
@@ -273,14 +273,14 @@ state:
switch (self->buffer[self->index]) {
case 0:
if (self->index == self->buffer_len) {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_INVALID;
@@ -293,16 +293,16 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '"':
result.tag = TOKENIZER_TAG_IDENTIFIER;
result.tag = TOKEN_IDENTIFIER;
state = TOKENIZER_STATE_STRING_LITERAL;
goto state;
case 'a' ... 'z':
case 'A' ... 'Z':
case '_':
result.tag = TOKENIZER_TAG_BUILTIN;
result.tag = TOKEN_BUILTIN;
state = TOKENIZER_STATE_BUILTIN;
goto state;
default:
@@ -315,11 +315,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_AMPERSAND_EQUAL;
result.tag = TOKEN_AMPERSAND_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_AMPERSAND;
result.tag = TOKEN_AMPERSAND;
break;
}
break;
@@ -328,11 +328,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ASTERISK_EQUAL;
result.tag = TOKEN_ASTERISK_EQUAL;
self->index++;
break;
case '*':
result.tag = TOKENIZER_TAG_ASTERISK_ASTERISK;
result.tag = TOKEN_ASTERISK_ASTERISK;
self->index++;
break;
case '%':
@@ -342,7 +342,7 @@ state:
state = TOKENIZER_STATE_ASTERISK_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_ASTERISK;
result.tag = TOKEN_ASTERISK;
break;
}
break;
@@ -351,11 +351,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL;
result.tag = TOKEN_ASTERISK_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ASTERISK_PERCENT;
result.tag = TOKEN_ASTERISK_PERCENT;
break;
}
break;
@@ -364,11 +364,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ASTERISK_PIPE_EQUAL;
result.tag = TOKEN_ASTERISK_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ASTERISK_PIPE;
result.tag = TOKEN_ASTERISK_PIPE;
break;
}
break;
@@ -377,11 +377,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PERCENT_EQUAL;
result.tag = TOKEN_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PERCENT;
result.tag = TOKEN_PERCENT;
break;
}
break;
@@ -390,11 +390,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PLUS_EQUAL;
result.tag = TOKEN_PLUS_EQUAL;
self->index++;
break;
case '+':
result.tag = TOKENIZER_TAG_PLUS_PLUS;
result.tag = TOKEN_PLUS_PLUS;
self->index++;
break;
case '%':
@@ -404,7 +404,7 @@ state:
state = TOKENIZER_STATE_PLUS_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_PLUS;
result.tag = TOKEN_PLUS;
break;
}
break;
@@ -413,11 +413,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PLUS_PERCENT_EQUAL;
result.tag = TOKEN_PLUS_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PLUS_PERCENT;
result.tag = TOKEN_PLUS_PERCENT;
break;
}
break;
@@ -426,11 +426,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PLUS_PIPE_EQUAL;
result.tag = TOKEN_PLUS_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PLUS_PIPE;
result.tag = TOKEN_PLUS_PIPE;
break;
}
break;
@@ -439,11 +439,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_CARET_EQUAL;
result.tag = TOKEN_CARET_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_CARET;
result.tag = TOKEN_CARET;
break;
}
break;
@@ -461,7 +461,7 @@ state:
const char* start = self->buffer + result.loc.start;
uint32_t len = self->index - result.loc.start;
TokenizerTag tag = getKeyword(start, len);
if (tag != TOKENIZER_TAG_INVALID) {
if (tag != TOKEN_INVALID) {
result.tag = tag;
}
}
@@ -484,13 +484,13 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case 0:
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_MULTILINE_STRING_LITERAL_LINE;
goto state;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_INVALID;
@@ -506,11 +506,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_STRING_LITERAL_BACKSLASH;
@@ -534,7 +534,7 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_STRING_LITERAL;
@@ -550,11 +550,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_CHAR_LITERAL_BACKSLASH;
@@ -581,11 +581,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case 0x01 ... 0x09:
case 0x0b ... 0x1f:
@@ -631,11 +631,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_BANG_EQUAL;
result.tag = TOKEN_BANG_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_BANG;
result.tag = TOKEN_BANG;
break;
}
break;
@@ -644,15 +644,15 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PIPE_EQUAL;
result.tag = TOKEN_PIPE_EQUAL;
self->index++;
break;
case '|':
result.tag = TOKENIZER_TAG_PIPE_PIPE;
result.tag = TOKEN_PIPE_PIPE;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PIPE;
result.tag = TOKEN_PIPE;
break;
}
break;
@@ -661,15 +661,15 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_EQUAL_EQUAL;
result.tag = TOKEN_EQUAL_EQUAL;
self->index++;
break;
case '>':
result.tag = TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT;
result.tag = TOKEN_EQUAL_ANGLE_BRACKET_RIGHT;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_EQUAL;
result.tag = TOKEN_EQUAL;
break;
}
break;
@@ -678,11 +678,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '>':
result.tag = TOKENIZER_TAG_ARROW;
result.tag = TOKEN_ARROW;
self->index++;
break;
case '=':
result.tag = TOKENIZER_TAG_MINUS_EQUAL;
result.tag = TOKEN_MINUS_EQUAL;
self->index++;
break;
case '%':
@@ -692,7 +692,7 @@ state:
state = TOKENIZER_STATE_MINUS_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_MINUS;
result.tag = TOKEN_MINUS;
break;
}
break;
@@ -701,11 +701,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_MINUS_PERCENT_EQUAL;
result.tag = TOKEN_MINUS_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_MINUS_PERCENT;
result.tag = TOKEN_MINUS_PERCENT;
break;
}
break;
@@ -714,11 +714,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_MINUS_PIPE_EQUAL;
result.tag = TOKEN_MINUS_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_MINUS_PIPE;
result.tag = TOKEN_MINUS_PIPE;
break;
}
break;
@@ -730,11 +730,11 @@ state:
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
goto state;
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_LEFT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_LEFT;
result.tag = TOKEN_ANGLE_BRACKET_LEFT;
break;
}
break;
@@ -743,14 +743,14 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL;
self->index++;
break;
case '|':
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
break;
}
break;
@@ -759,11 +759,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
break;
}
break;
@@ -775,11 +775,11 @@ state:
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
goto state;
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_RIGHT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_RIGHT;
result.tag = TOKEN_ANGLE_BRACKET_RIGHT;
break;
}
break;
@@ -788,11 +788,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
break;
}
break;
@@ -807,7 +807,7 @@ state:
state = TOKENIZER_STATE_PERIOD_ASTERISK;
goto state;
default:
result.tag = TOKENIZER_TAG_PERIOD;
result.tag = TOKEN_PERIOD;
break;
}
break;
@@ -816,11 +816,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '.':
result.tag = TOKENIZER_TAG_ELLIPSIS3;
result.tag = TOKEN_ELLIPSIS3;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ELLIPSIS2;
result.tag = TOKEN_ELLIPSIS2;
break;
}
break;
@@ -829,10 +829,10 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '*':
result.tag = TOKENIZER_TAG_INVALID_PERIODASTERISKS;
result.tag = TOKEN_INVALID_PERIODASTERISKS;
break;
default:
result.tag = TOKENIZER_TAG_PERIOD_ASTERISK;
result.tag = TOKEN_PERIOD_ASTERISK;
break;
}
break;
@@ -844,11 +844,11 @@ state:
state = TOKENIZER_STATE_LINE_COMMENT_START;
goto state;
case '=':
result.tag = TOKENIZER_TAG_SLASH_EQUAL;
result.tag = TOKEN_SLASH_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_SLASH;
result.tag = TOKEN_SLASH;
break;
}
break;
@@ -862,7 +862,7 @@ state:
goto state;
} else {
return (TokenizerToken) {
.tag = TOKENIZER_TAG_EOF,
.tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index }
@@ -870,7 +870,7 @@ state:
}
break;
case '!':
result.tag = TOKENIZER_TAG_CONTAINER_DOC_COMMENT;
result.tag = TOKEN_CONTAINER_DOC_COMMENT;
state = TOKENIZER_STATE_DOC_COMMENT;
goto state;
case '\n':
@@ -901,11 +901,11 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
result.tag = TOKENIZER_TAG_DOC_COMMENT;
result.tag = TOKEN_DOC_COMMENT;
break;
case '\r':
if (self->buffer[self->index + 1] == '\n') {
result.tag = TOKENIZER_TAG_DOC_COMMENT;
result.tag = TOKEN_DOC_COMMENT;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
@@ -921,7 +921,7 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
default:
result.tag = TOKENIZER_TAG_DOC_COMMENT;
result.tag = TOKEN_DOC_COMMENT;
state = TOKENIZER_STATE_DOC_COMMENT;
goto state;
}
@@ -936,7 +936,7 @@ state:
goto state;
} else {
return (TokenizerToken) {
.tag = TOKENIZER_TAG_EOF,
.tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index }

244
tokenizer.h
View File

@@ -5,128 +5,128 @@
#include <stdint.h>
#define TOKENIZER_FOREACH_TAG_ENUM(TAG) \
TAG(TOKENIZER_TAG_INVALID) \
TAG(TOKENIZER_TAG_INVALID_PERIODASTERISKS) \
TAG(TOKENIZER_TAG_IDENTIFIER) \
TAG(TOKENIZER_TAG_STRING_LITERAL) \
TAG(TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE) \
TAG(TOKENIZER_TAG_CHAR_LITERAL) \
TAG(TOKENIZER_TAG_EOF) \
TAG(TOKENIZER_TAG_BUILTIN) \
TAG(TOKENIZER_TAG_BANG) \
TAG(TOKENIZER_TAG_PIPE) \
TAG(TOKENIZER_TAG_PIPE_PIPE) \
TAG(TOKENIZER_TAG_PIPE_EQUAL) \
TAG(TOKENIZER_TAG_EQUAL) \
TAG(TOKENIZER_TAG_EQUAL_EQUAL) \
TAG(TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT) \
TAG(TOKENIZER_TAG_BANG_EQUAL) \
TAG(TOKENIZER_TAG_L_PAREN) \
TAG(TOKENIZER_TAG_R_PAREN) \
TAG(TOKENIZER_TAG_SEMICOLON) \
TAG(TOKENIZER_TAG_PERCENT) \
TAG(TOKENIZER_TAG_PERCENT_EQUAL) \
TAG(TOKENIZER_TAG_L_BRACE) \
TAG(TOKENIZER_TAG_R_BRACE) \
TAG(TOKENIZER_TAG_L_BRACKET) \
TAG(TOKENIZER_TAG_R_BRACKET) \
TAG(TOKENIZER_TAG_PERIOD) \
TAG(TOKENIZER_TAG_PERIOD_ASTERISK) \
TAG(TOKENIZER_TAG_ELLIPSIS2) \
TAG(TOKENIZER_TAG_ELLIPSIS3) \
TAG(TOKENIZER_TAG_CARET) \
TAG(TOKENIZER_TAG_CARET_EQUAL) \
TAG(TOKENIZER_TAG_PLUS) \
TAG(TOKENIZER_TAG_PLUS_PLUS) \
TAG(TOKENIZER_TAG_PLUS_EQUAL) \
TAG(TOKENIZER_TAG_PLUS_PERCENT) \
TAG(TOKENIZER_TAG_PLUS_PERCENT_EQUAL) \
TAG(TOKENIZER_TAG_PLUS_PIPE) \
TAG(TOKENIZER_TAG_PLUS_PIPE_EQUAL) \
TAG(TOKENIZER_TAG_MINUS) \
TAG(TOKENIZER_TAG_MINUS_EQUAL) \
TAG(TOKENIZER_TAG_MINUS_PERCENT) \
TAG(TOKENIZER_TAG_MINUS_PERCENT_EQUAL) \
TAG(TOKENIZER_TAG_MINUS_PIPE) \
TAG(TOKENIZER_TAG_MINUS_PIPE_EQUAL) \
TAG(TOKENIZER_TAG_ASTERISK) \
TAG(TOKENIZER_TAG_ASTERISK_EQUAL) \
TAG(TOKENIZER_TAG_ASTERISK_ASTERISK) \
TAG(TOKENIZER_TAG_ASTERISK_PERCENT) \
TAG(TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL) \
TAG(TOKENIZER_TAG_ASTERISK_PIPE) \
TAG(TOKENIZER_TAG_ASTERISK_PIPE_EQUAL) \
TAG(TOKENIZER_TAG_ARROW) \
TAG(TOKENIZER_TAG_COLON) \
TAG(TOKENIZER_TAG_SLASH) \
TAG(TOKENIZER_TAG_SLASH_EQUAL) \
TAG(TOKENIZER_TAG_COMMA) \
TAG(TOKENIZER_TAG_AMPERSAND) \
TAG(TOKENIZER_TAG_AMPERSAND_EQUAL) \
TAG(TOKENIZER_TAG_QUESTION_MARK) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_LEFT) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_RIGHT) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT) \
TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL) \
TAG(TOKENIZER_TAG_TILDE) \
TAG(TOKENIZER_TAG_NUMBER_LITERAL) \
TAG(TOKENIZER_TAG_DOC_COMMENT) \
TAG(TOKENIZER_TAG_CONTAINER_DOC_COMMENT) \
TAG(TOKENIZER_TAG_KEYWORD_ADDRSPACE) \
TAG(TOKENIZER_TAG_KEYWORD_ALIGN) \
TAG(TOKENIZER_TAG_KEYWORD_ALLOWZERO) \
TAG(TOKENIZER_TAG_KEYWORD_AND) \
TAG(TOKENIZER_TAG_KEYWORD_ANYFRAME) \
TAG(TOKENIZER_TAG_KEYWORD_ANYTYPE) \
TAG(TOKENIZER_TAG_KEYWORD_ASM) \
TAG(TOKENIZER_TAG_KEYWORD_ASYNC) \
TAG(TOKENIZER_TAG_KEYWORD_AWAIT) \
TAG(TOKENIZER_TAG_KEYWORD_BREAK) \
TAG(TOKENIZER_TAG_KEYWORD_CALLCONV) \
TAG(TOKENIZER_TAG_KEYWORD_CATCH) \
TAG(TOKENIZER_TAG_KEYWORD_COMPTIME) \
TAG(TOKENIZER_TAG_KEYWORD_CONST) \
TAG(TOKENIZER_TAG_KEYWORD_CONTINUE) \
TAG(TOKENIZER_TAG_KEYWORD_DEFER) \
TAG(TOKENIZER_TAG_KEYWORD_ELSE) \
TAG(TOKENIZER_TAG_KEYWORD_ENUM) \
TAG(TOKENIZER_TAG_KEYWORD_ERRDEFER) \
TAG(TOKENIZER_TAG_KEYWORD_ERROR) \
TAG(TOKENIZER_TAG_KEYWORD_EXPORT) \
TAG(TOKENIZER_TAG_KEYWORD_EXTERN) \
TAG(TOKENIZER_TAG_KEYWORD_FN) \
TAG(TOKENIZER_TAG_KEYWORD_FOR) \
TAG(TOKENIZER_TAG_KEYWORD_IF) \
TAG(TOKENIZER_TAG_KEYWORD_INLINE) \
TAG(TOKENIZER_TAG_KEYWORD_NOALIAS) \
TAG(TOKENIZER_TAG_KEYWORD_NOINLINE) \
TAG(TOKENIZER_TAG_KEYWORD_NOSUSPEND) \
TAG(TOKENIZER_TAG_KEYWORD_OPAQUE) \
TAG(TOKENIZER_TAG_KEYWORD_OR) \
TAG(TOKENIZER_TAG_KEYWORD_ORELSE) \
TAG(TOKENIZER_TAG_KEYWORD_PACKED) \
TAG(TOKENIZER_TAG_KEYWORD_PUB) \
TAG(TOKENIZER_TAG_KEYWORD_RESUME) \
TAG(TOKENIZER_TAG_KEYWORD_RETURN) \
TAG(TOKENIZER_TAG_KEYWORD_LINKSECTION) \
TAG(TOKENIZER_TAG_KEYWORD_STRUCT) \
TAG(TOKENIZER_TAG_KEYWORD_SUSPEND) \
TAG(TOKENIZER_TAG_KEYWORD_SWITCH) \
TAG(TOKENIZER_TAG_KEYWORD_TEST) \
TAG(TOKENIZER_TAG_KEYWORD_THREADLOCAL) \
TAG(TOKENIZER_TAG_KEYWORD_TRY) \
TAG(TOKENIZER_TAG_KEYWORD_UNION) \
TAG(TOKENIZER_TAG_KEYWORD_UNREACHABLE) \
TAG(TOKENIZER_TAG_KEYWORD_USINGNAMESPACE) \
TAG(TOKENIZER_TAG_KEYWORD_VAR) \
TAG(TOKENIZER_TAG_KEYWORD_VOLATILE) \
TAG(TOKENIZER_TAG_KEYWORD_WHILE)
TAG(TOKEN_INVALID) \
TAG(TOKEN_INVALID_PERIODASTERISKS) \
TAG(TOKEN_IDENTIFIER) \
TAG(TOKEN_STRING_LITERAL) \
TAG(TOKEN_MULTILINE_STRING_LITERAL_LINE) \
TAG(TOKEN_CHAR_LITERAL) \
TAG(TOKEN_EOF) \
TAG(TOKEN_BUILTIN) \
TAG(TOKEN_BANG) \
TAG(TOKEN_PIPE) \
TAG(TOKEN_PIPE_PIPE) \
TAG(TOKEN_PIPE_EQUAL) \
TAG(TOKEN_EQUAL) \
TAG(TOKEN_EQUAL_EQUAL) \
TAG(TOKEN_EQUAL_ANGLE_BRACKET_RIGHT) \
TAG(TOKEN_BANG_EQUAL) \
TAG(TOKEN_L_PAREN) \
TAG(TOKEN_R_PAREN) \
TAG(TOKEN_SEMICOLON) \
TAG(TOKEN_PERCENT) \
TAG(TOKEN_PERCENT_EQUAL) \
TAG(TOKEN_L_BRACE) \
TAG(TOKEN_R_BRACE) \
TAG(TOKEN_L_BRACKET) \
TAG(TOKEN_R_BRACKET) \
TAG(TOKEN_PERIOD) \
TAG(TOKEN_PERIOD_ASTERISK) \
TAG(TOKEN_ELLIPSIS2) \
TAG(TOKEN_ELLIPSIS3) \
TAG(TOKEN_CARET) \
TAG(TOKEN_CARET_EQUAL) \
TAG(TOKEN_PLUS) \
TAG(TOKEN_PLUS_PLUS) \
TAG(TOKEN_PLUS_EQUAL) \
TAG(TOKEN_PLUS_PERCENT) \
TAG(TOKEN_PLUS_PERCENT_EQUAL) \
TAG(TOKEN_PLUS_PIPE) \
TAG(TOKEN_PLUS_PIPE_EQUAL) \
TAG(TOKEN_MINUS) \
TAG(TOKEN_MINUS_EQUAL) \
TAG(TOKEN_MINUS_PERCENT) \
TAG(TOKEN_MINUS_PERCENT_EQUAL) \
TAG(TOKEN_MINUS_PIPE) \
TAG(TOKEN_MINUS_PIPE_EQUAL) \
TAG(TOKEN_ASTERISK) \
TAG(TOKEN_ASTERISK_EQUAL) \
TAG(TOKEN_ASTERISK_ASTERISK) \
TAG(TOKEN_ASTERISK_PERCENT) \
TAG(TOKEN_ASTERISK_PERCENT_EQUAL) \
TAG(TOKEN_ASTERISK_PIPE) \
TAG(TOKEN_ASTERISK_PIPE_EQUAL) \
TAG(TOKEN_ARROW) \
TAG(TOKEN_COLON) \
TAG(TOKEN_SLASH) \
TAG(TOKEN_SLASH_EQUAL) \
TAG(TOKEN_COMMA) \
TAG(TOKEN_AMPERSAND) \
TAG(TOKEN_AMPERSAND_EQUAL) \
TAG(TOKEN_QUESTION_MARK) \
TAG(TOKEN_ANGLE_BRACKET_LEFT) \
TAG(TOKEN_ANGLE_BRACKET_LEFT_EQUAL) \
TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT) \
TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL) \
TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE) \
TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL) \
TAG(TOKEN_ANGLE_BRACKET_RIGHT) \
TAG(TOKEN_ANGLE_BRACKET_RIGHT_EQUAL) \
TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT) \
TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL) \
TAG(TOKEN_TILDE) \
TAG(TOKEN_NUMBER_LITERAL) \
TAG(TOKEN_DOC_COMMENT) \
TAG(TOKEN_CONTAINER_DOC_COMMENT) \
TAG(TOKEN_KEYWORD_ADDRSPACE) \
TAG(TOKEN_KEYWORD_ALIGN) \
TAG(TOKEN_KEYWORD_ALLOWZERO) \
TAG(TOKEN_KEYWORD_AND) \
TAG(TOKEN_KEYWORD_ANYFRAME) \
TAG(TOKEN_KEYWORD_ANYTYPE) \
TAG(TOKEN_KEYWORD_ASM) \
TAG(TOKEN_KEYWORD_ASYNC) \
TAG(TOKEN_KEYWORD_AWAIT) \
TAG(TOKEN_KEYWORD_BREAK) \
TAG(TOKEN_KEYWORD_CALLCONV) \
TAG(TOKEN_KEYWORD_CATCH) \
TAG(TOKEN_KEYWORD_COMPTIME) \
TAG(TOKEN_KEYWORD_CONST) \
TAG(TOKEN_KEYWORD_CONTINUE) \
TAG(TOKEN_KEYWORD_DEFER) \
TAG(TOKEN_KEYWORD_ELSE) \
TAG(TOKEN_KEYWORD_ENUM) \
TAG(TOKEN_KEYWORD_ERRDEFER) \
TAG(TOKEN_KEYWORD_ERROR) \
TAG(TOKEN_KEYWORD_EXPORT) \
TAG(TOKEN_KEYWORD_EXTERN) \
TAG(TOKEN_KEYWORD_FN) \
TAG(TOKEN_KEYWORD_FOR) \
TAG(TOKEN_KEYWORD_IF) \
TAG(TOKEN_KEYWORD_INLINE) \
TAG(TOKEN_KEYWORD_NOALIAS) \
TAG(TOKEN_KEYWORD_NOINLINE) \
TAG(TOKEN_KEYWORD_NOSUSPEND) \
TAG(TOKEN_KEYWORD_OPAQUE) \
TAG(TOKEN_KEYWORD_OR) \
TAG(TOKEN_KEYWORD_ORELSE) \
TAG(TOKEN_KEYWORD_PACKED) \
TAG(TOKEN_KEYWORD_PUB) \
TAG(TOKEN_KEYWORD_RESUME) \
TAG(TOKEN_KEYWORD_RETURN) \
TAG(TOKEN_KEYWORD_LINKSECTION) \
TAG(TOKEN_KEYWORD_STRUCT) \
TAG(TOKEN_KEYWORD_SUSPEND) \
TAG(TOKEN_KEYWORD_SWITCH) \
TAG(TOKEN_KEYWORD_TEST) \
TAG(TOKEN_KEYWORD_THREADLOCAL) \
TAG(TOKEN_KEYWORD_TRY) \
TAG(TOKEN_KEYWORD_UNION) \
TAG(TOKEN_KEYWORD_UNREACHABLE) \
TAG(TOKEN_KEYWORD_USINGNAMESPACE) \
TAG(TOKEN_KEYWORD_VAR) \
TAG(TOKEN_KEYWORD_VOLATILE) \
TAG(TOKEN_KEYWORD_WHILE)
#define TOKENIZER_GENERATE_ENUM(ENUM) ENUM,
#define TOKENIZER_GENERATE_CASE(ENUM) \

244
tokenizer_test.zig
View File

@@ -10,128 +10,128 @@ const c = @cImport({
fn zigToken(token: c_uint) Token.Tag {
return switch (token) {
c.TOKENIZER_TAG_INVALID => .invalid,
c.TOKENIZER_TAG_INVALID_PERIODASTERISKS => .invalid_periodasterisks,
c.TOKENIZER_TAG_IDENTIFIER => .identifier,
c.TOKENIZER_TAG_STRING_LITERAL => .string_literal,
c.TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE => .multiline_string_literal_line,
c.TOKENIZER_TAG_CHAR_LITERAL => .char_literal,
c.TOKENIZER_TAG_EOF => .eof,
c.TOKENIZER_TAG_BUILTIN => .builtin,
c.TOKENIZER_TAG_BANG => .bang,
c.TOKENIZER_TAG_PIPE => .pipe,
c.TOKENIZER_TAG_PIPE_PIPE => .pipe_pipe,
c.TOKENIZER_TAG_PIPE_EQUAL => .pipe_equal,
c.TOKENIZER_TAG_EQUAL => .equal,
c.TOKENIZER_TAG_EQUAL_EQUAL => .equal_equal,
c.TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT => .equal_angle_bracket_right,
c.TOKENIZER_TAG_BANG_EQUAL => .bang_equal,
c.TOKENIZER_TAG_L_PAREN => .l_paren,
c.TOKENIZER_TAG_R_PAREN => .r_paren,
c.TOKENIZER_TAG_SEMICOLON => .semicolon,
c.TOKENIZER_TAG_PERCENT => .percent,
c.TOKENIZER_TAG_PERCENT_EQUAL => .percent_equal,
c.TOKENIZER_TAG_L_BRACE => .l_brace,
c.TOKENIZER_TAG_R_BRACE => .r_brace,
c.TOKENIZER_TAG_L_BRACKET => .l_bracket,
c.TOKENIZER_TAG_R_BRACKET => .r_bracket,
c.TOKENIZER_TAG_PERIOD => .period,
c.TOKENIZER_TAG_PERIOD_ASTERISK => .period_asterisk,
c.TOKENIZER_TAG_ELLIPSIS2 => .ellipsis2,
c.TOKENIZER_TAG_ELLIPSIS3 => .ellipsis3,
c.TOKENIZER_TAG_CARET => .caret,
c.TOKENIZER_TAG_CARET_EQUAL => .caret_equal,
c.TOKENIZER_TAG_PLUS => .plus,
c.TOKENIZER_TAG_PLUS_PLUS => .plus_plus,
c.TOKENIZER_TAG_PLUS_EQUAL => .plus_equal,
c.TOKENIZER_TAG_PLUS_PERCENT => .plus_percent,
c.TOKENIZER_TAG_PLUS_PERCENT_EQUAL => .plus_percent_equal,
c.TOKENIZER_TAG_PLUS_PIPE => .plus_pipe,
c.TOKENIZER_TAG_PLUS_PIPE_EQUAL => .plus_pipe_equal,
c.TOKENIZER_TAG_MINUS => .minus,
c.TOKENIZER_TAG_MINUS_EQUAL => .minus_equal,
c.TOKENIZER_TAG_MINUS_PERCENT => .minus_percent,
c.TOKENIZER_TAG_MINUS_PERCENT_EQUAL => .minus_percent_equal,
c.TOKENIZER_TAG_MINUS_PIPE => .minus_pipe,
c.TOKENIZER_TAG_MINUS_PIPE_EQUAL => .minus_pipe_equal,
c.TOKENIZER_TAG_ASTERISK => .asterisk,
c.TOKENIZER_TAG_ASTERISK_EQUAL => .asterisk_equal,
c.TOKENIZER_TAG_ASTERISK_ASTERISK => .asterisk_asterisk,
c.TOKENIZER_TAG_ASTERISK_PERCENT => .asterisk_percent,
c.TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL => .asterisk_percent_equal,
c.TOKENIZER_TAG_ASTERISK_PIPE => .asterisk_pipe,
c.TOKENIZER_TAG_ASTERISK_PIPE_EQUAL => .asterisk_pipe_equal,
c.TOKENIZER_TAG_ARROW => .arrow,
c.TOKENIZER_TAG_COLON => .colon,
c.TOKENIZER_TAG_SLASH => .slash,
c.TOKENIZER_TAG_SLASH_EQUAL => .slash_equal,
c.TOKENIZER_TAG_COMMA => .comma,
c.TOKENIZER_TAG_AMPERSAND => .ampersand,
c.TOKENIZER_TAG_AMPERSAND_EQUAL => .ampersand_equal,
c.TOKENIZER_TAG_QUESTION_MARK => .question_mark,
c.TOKENIZER_TAG_ANGLE_BRACKET_LEFT => .angle_bracket_left,
c.TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_left_equal,
c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT => .angle_bracket_angle_bracket_left,
c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_angle_bracket_left_equal,
c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE => .angle_bracket_angle_bracket_left_pipe,
c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL => .angle_bracket_angle_bracket_left_pipe_equal,
c.TOKENIZER_TAG_ANGLE_BRACKET_RIGHT => .angle_bracket_right,
c.TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_right_equal,
c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT => .angle_bracket_angle_bracket_right,
c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_angle_bracket_right_equal,
c.TOKENIZER_TAG_TILDE => .tilde,
c.TOKENIZER_TAG_NUMBER_LITERAL => .number_literal,
c.TOKENIZER_TAG_DOC_COMMENT => .doc_comment,
c.TOKENIZER_TAG_CONTAINER_DOC_COMMENT => .container_doc_comment,
c.TOKENIZER_TAG_KEYWORD_ADDRSPACE => .keyword_addrspace,
c.TOKENIZER_TAG_KEYWORD_ALIGN => .keyword_align,
c.TOKENIZER_TAG_KEYWORD_ALLOWZERO => .keyword_allowzero,
c.TOKENIZER_TAG_KEYWORD_AND => .keyword_and,
c.TOKENIZER_TAG_KEYWORD_ANYFRAME => .keyword_anyframe,
c.TOKENIZER_TAG_KEYWORD_ANYTYPE => .keyword_anytype,
c.TOKENIZER_TAG_KEYWORD_ASM => .keyword_asm,
c.TOKENIZER_TAG_KEYWORD_ASYNC => .keyword_async,
c.TOKENIZER_TAG_KEYWORD_AWAIT => .keyword_await,
c.TOKENIZER_TAG_KEYWORD_BREAK => .keyword_break,
c.TOKENIZER_TAG_KEYWORD_CALLCONV => .keyword_callconv,
c.TOKENIZER_TAG_KEYWORD_CATCH => .keyword_catch,
c.TOKENIZER_TAG_KEYWORD_COMPTIME => .keyword_comptime,
c.TOKENIZER_TAG_KEYWORD_CONST => .keyword_const,
c.TOKENIZER_TAG_KEYWORD_CONTINUE => .keyword_continue,
c.TOKENIZER_TAG_KEYWORD_DEFER => .keyword_defer,
c.TOKENIZER_TAG_KEYWORD_ELSE => .keyword_else,
c.TOKENIZER_TAG_KEYWORD_ENUM => .keyword_enum,
c.TOKENIZER_TAG_KEYWORD_ERRDEFER => .keyword_errdefer,
c.TOKENIZER_TAG_KEYWORD_ERROR => .keyword_error,
c.TOKENIZER_TAG_KEYWORD_EXPORT => .keyword_export,
c.TOKENIZER_TAG_KEYWORD_EXTERN => .keyword_extern,
c.TOKENIZER_TAG_KEYWORD_FN => .keyword_fn,
c.TOKENIZER_TAG_KEYWORD_FOR => .keyword_for,
c.TOKENIZER_TAG_KEYWORD_IF => .keyword_if,
c.TOKENIZER_TAG_KEYWORD_INLINE => .keyword_inline,
c.TOKENIZER_TAG_KEYWORD_NOALIAS => .keyword_noalias,
c.TOKENIZER_TAG_KEYWORD_NOINLINE => .keyword_noinline,
c.TOKENIZER_TAG_KEYWORD_NOSUSPEND => .keyword_nosuspend,
c.TOKENIZER_TAG_KEYWORD_OPAQUE => .keyword_opaque,
c.TOKENIZER_TAG_KEYWORD_OR => .keyword_or,
c.TOKENIZER_TAG_KEYWORD_ORELSE => .keyword_orelse,
c.TOKENIZER_TAG_KEYWORD_PACKED => .keyword_packed,
c.TOKENIZER_TAG_KEYWORD_PUB => .keyword_pub,
c.TOKENIZER_TAG_KEYWORD_RESUME => .keyword_resume,
c.TOKENIZER_TAG_KEYWORD_RETURN => .keyword_return,
c.TOKENIZER_TAG_KEYWORD_LINKSECTION => .keyword_linksection,
c.TOKENIZER_TAG_KEYWORD_STRUCT => .keyword_struct,
c.TOKENIZER_TAG_KEYWORD_SUSPEND => .keyword_suspend,
c.TOKENIZER_TAG_KEYWORD_SWITCH => .keyword_switch,
c.TOKENIZER_TAG_KEYWORD_TEST => .keyword_test,
c.TOKENIZER_TAG_KEYWORD_THREADLOCAL => .keyword_threadlocal,
c.TOKENIZER_TAG_KEYWORD_TRY => .keyword_try,
c.TOKENIZER_TAG_KEYWORD_UNION => .keyword_union,
c.TOKENIZER_TAG_KEYWORD_UNREACHABLE => .keyword_unreachable,
c.TOKENIZER_TAG_KEYWORD_USINGNAMESPACE => .keyword_usingnamespace,
c.TOKENIZER_TAG_KEYWORD_VAR => .keyword_var,
c.TOKENIZER_TAG_KEYWORD_VOLATILE => .keyword_volatile,
c.TOKENIZER_TAG_KEYWORD_WHILE => .keyword_while,
c.TOKEN_INVALID => .invalid,
c.TOKEN_INVALID_PERIODASTERISKS => .invalid_periodasterisks,
c.TOKEN_IDENTIFIER => .identifier,
c.TOKEN_STRING_LITERAL => .string_literal,
c.TOKEN_MULTILINE_STRING_LITERAL_LINE => .multiline_string_literal_line,
c.TOKEN_CHAR_LITERAL => .char_literal,
c.TOKEN_EOF => .eof,
c.TOKEN_BUILTIN => .builtin,
c.TOKEN_BANG => .bang,
c.TOKEN_PIPE => .pipe,
c.TOKEN_PIPE_PIPE => .pipe_pipe,
c.TOKEN_PIPE_EQUAL => .pipe_equal,
c.TOKEN_EQUAL => .equal,
c.TOKEN_EQUAL_EQUAL => .equal_equal,
c.TOKEN_EQUAL_ANGLE_BRACKET_RIGHT => .equal_angle_bracket_right,
c.TOKEN_BANG_EQUAL => .bang_equal,
c.TOKEN_L_PAREN => .l_paren,
c.TOKEN_R_PAREN => .r_paren,
c.TOKEN_SEMICOLON => .semicolon,
c.TOKEN_PERCENT => .percent,
c.TOKEN_PERCENT_EQUAL => .percent_equal,
c.TOKEN_L_BRACE => .l_brace,
c.TOKEN_R_BRACE => .r_brace,
c.TOKEN_L_BRACKET => .l_bracket,
c.TOKEN_R_BRACKET => .r_bracket,
c.TOKEN_PERIOD => .period,
c.TOKEN_PERIOD_ASTERISK => .period_asterisk,
c.TOKEN_ELLIPSIS2 => .ellipsis2,
c.TOKEN_ELLIPSIS3 => .ellipsis3,
c.TOKEN_CARET => .caret,
c.TOKEN_CARET_EQUAL => .caret_equal,
c.TOKEN_PLUS => .plus,
c.TOKEN_PLUS_PLUS => .plus_plus,
c.TOKEN_PLUS_EQUAL => .plus_equal,
c.TOKEN_PLUS_PERCENT => .plus_percent,
c.TOKEN_PLUS_PERCENT_EQUAL => .plus_percent_equal,
c.TOKEN_PLUS_PIPE => .plus_pipe,
c.TOKEN_PLUS_PIPE_EQUAL => .plus_pipe_equal,
c.TOKEN_MINUS => .minus,
c.TOKEN_MINUS_EQUAL => .minus_equal,
c.TOKEN_MINUS_PERCENT => .minus_percent,
c.TOKEN_MINUS_PERCENT_EQUAL => .minus_percent_equal,
c.TOKEN_MINUS_PIPE => .minus_pipe,
c.TOKEN_MINUS_PIPE_EQUAL => .minus_pipe_equal,
c.TOKEN_ASTERISK => .asterisk,
c.TOKEN_ASTERISK_EQUAL => .asterisk_equal,
c.TOKEN_ASTERISK_ASTERISK => .asterisk_asterisk,
c.TOKEN_ASTERISK_PERCENT => .asterisk_percent,
c.TOKEN_ASTERISK_PERCENT_EQUAL => .asterisk_percent_equal,
c.TOKEN_ASTERISK_PIPE => .asterisk_pipe,
c.TOKEN_ASTERISK_PIPE_EQUAL => .asterisk_pipe_equal,
c.TOKEN_ARROW => .arrow,
c.TOKEN_COLON => .colon,
c.TOKEN_SLASH => .slash,
c.TOKEN_SLASH_EQUAL => .slash_equal,
c.TOKEN_COMMA => .comma,
c.TOKEN_AMPERSAND => .ampersand,
c.TOKEN_AMPERSAND_EQUAL => .ampersand_equal,
c.TOKEN_QUESTION_MARK => .question_mark,
c.TOKEN_ANGLE_BRACKET_LEFT => .angle_bracket_left,
c.TOKEN_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_left_equal,
c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT => .angle_bracket_angle_bracket_left,
c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_angle_bracket_left_equal,
c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE => .angle_bracket_angle_bracket_left_pipe,
c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL => .angle_bracket_angle_bracket_left_pipe_equal,
c.TOKEN_ANGLE_BRACKET_RIGHT => .angle_bracket_right,
c.TOKEN_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_right_equal,
c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT => .angle_bracket_angle_bracket_right,
c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_angle_bracket_right_equal,
c.TOKEN_TILDE => .tilde,
c.TOKEN_NUMBER_LITERAL => .number_literal,
c.TOKEN_DOC_COMMENT => .doc_comment,
c.TOKEN_CONTAINER_DOC_COMMENT => .container_doc_comment,
c.TOKEN_KEYWORD_ADDRSPACE => .keyword_addrspace,
c.TOKEN_KEYWORD_ALIGN => .keyword_align,
c.TOKEN_KEYWORD_ALLOWZERO => .keyword_allowzero,
c.TOKEN_KEYWORD_AND => .keyword_and,
c.TOKEN_KEYWORD_ANYFRAME => .keyword_anyframe,
c.TOKEN_KEYWORD_ANYTYPE => .keyword_anytype,
c.TOKEN_KEYWORD_ASM => .keyword_asm,
c.TOKEN_KEYWORD_ASYNC => .keyword_async,
c.TOKEN_KEYWORD_AWAIT => .keyword_await,
c.TOKEN_KEYWORD_BREAK => .keyword_break,
c.TOKEN_KEYWORD_CALLCONV => .keyword_callconv,
c.TOKEN_KEYWORD_CATCH => .keyword_catch,
c.TOKEN_KEYWORD_COMPTIME => .keyword_comptime,
c.TOKEN_KEYWORD_CONST => .keyword_const,
c.TOKEN_KEYWORD_CONTINUE => .keyword_continue,
c.TOKEN_KEYWORD_DEFER => .keyword_defer,
c.TOKEN_KEYWORD_ELSE => .keyword_else,
c.TOKEN_KEYWORD_ENUM => .keyword_enum,
c.TOKEN_KEYWORD_ERRDEFER => .keyword_errdefer,
c.TOKEN_KEYWORD_ERROR => .keyword_error,
c.TOKEN_KEYWORD_EXPORT => .keyword_export,
c.TOKEN_KEYWORD_EXTERN => .keyword_extern,
c.TOKEN_KEYWORD_FN => .keyword_fn,
c.TOKEN_KEYWORD_FOR => .keyword_for,
c.TOKEN_KEYWORD_IF => .keyword_if,
c.TOKEN_KEYWORD_INLINE => .keyword_inline,
c.TOKEN_KEYWORD_NOALIAS => .keyword_noalias,
c.TOKEN_KEYWORD_NOINLINE => .keyword_noinline,
c.TOKEN_KEYWORD_NOSUSPEND => .keyword_nosuspend,
c.TOKEN_KEYWORD_OPAQUE => .keyword_opaque,
c.TOKEN_KEYWORD_OR => .keyword_or,
c.TOKEN_KEYWORD_ORELSE => .keyword_orelse,
c.TOKEN_KEYWORD_PACKED => .keyword_packed,
c.TOKEN_KEYWORD_PUB => .keyword_pub,
c.TOKEN_KEYWORD_RESUME => .keyword_resume,
c.TOKEN_KEYWORD_RETURN => .keyword_return,
c.TOKEN_KEYWORD_LINKSECTION => .keyword_linksection,
c.TOKEN_KEYWORD_STRUCT => .keyword_struct,
c.TOKEN_KEYWORD_SUSPEND => .keyword_suspend,
c.TOKEN_KEYWORD_SWITCH => .keyword_switch,
c.TOKEN_KEYWORD_TEST => .keyword_test,
c.TOKEN_KEYWORD_THREADLOCAL => .keyword_threadlocal,
c.TOKEN_KEYWORD_TRY => .keyword_try,
c.TOKEN_KEYWORD_UNION => .keyword_union,
c.TOKEN_KEYWORD_UNREACHABLE => .keyword_unreachable,
c.TOKEN_KEYWORD_USINGNAMESPACE => .keyword_usingnamespace,
c.TOKEN_KEYWORD_VAR => .keyword_var,
c.TOKEN_KEYWORD_VOLATILE => .keyword_volatile,
c.TOKEN_KEYWORD_WHILE => .keyword_while,
else => undefined,
};
}