commit 3264d1747ed17872d36638e3f94729a9eb14b8ef (tree)
parent d551ba3d12515b38d409d84f7f56ff64bc044301
Author: Motiejus Jakštys <motiejus@jakstys.lt>
Date: Mon, 30 Dec 2024 22:38:17 +0200
replace TOKENIZER_TAG_ with TOKEN_
Diffstat:
| M | ast.c | | | 2 | +- |
| M | parser.c | | | 244 | ++++++++++++++++++++++++++++++++++++++++---------------------------------------- |
| M | tokenizer.c | | | 288 | ++++++++++++++++++++++++++++++++++++++++---------------------------------------- |
| M | tokenizer.h | | | 244 | ++++++++++++++++++++++++++++++++++++++++---------------------------------------- |
| M | tokenizer_test.zig | | | 244 | ++++++++++++++++++++++++++++++++++++++++---------------------------------------- |
5 files changed, 511 insertions(+), 511 deletions(-)
diff --git a/ast.c b/ast.c
@@ -31,7 +31,7 @@ Ast astParse(const char* source, const uint32_t len) {
tokens.tags[tokens.len] = token.tag;
tokens.starts[tokens.len] = token.loc.start;
tokens.len++;
- if (token.tag == TOKENIZER_TAG_EOF)
+ if (token.tag == TOKEN_EOF)
break;
}
diff --git a/parser.c b/parser.c
@@ -74,12 +74,12 @@ static AstTokenIndex eatToken(Parser* p, TokenizerTag tag, bool* ok) {
static void eatDocComments(Parser* p) {
bool ok;
- while (eatToken(p, TOKENIZER_TAG_DOC_COMMENT, &ok), ok) { }
+ while (eatToken(p, TOKEN_DOC_COMMENT, &ok), ok) { }
}
static void expectSemicolon(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_SEMICOLON, &ok);
+ eatToken(p, TOKEN_SEMICOLON, &ok);
if (ok)
return;
@@ -126,7 +126,7 @@ static AstNodeIndex addExtra(Parser* p, const AstNodeIndex* extra, uint32_t coun
static AstNodeIndex parseByteAlign(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_KEYWORD_ALIGN, &ok);
+ eatToken(p, TOKEN_KEYWORD_ALIGN, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseByteAlign cannot parse alginment\n");
@@ -136,7 +136,7 @@ static AstNodeIndex parseByteAlign(Parser* p) {
static AstNodeIndex parseAddrSpace(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_KEYWORD_ADDRSPACE, &ok);
+ eatToken(p, TOKEN_KEYWORD_ADDRSPACE, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseAddrSpace cannot parse addrspace\n");
@@ -146,7 +146,7 @@ static AstNodeIndex parseAddrSpace(Parser* p) {
static AstNodeIndex parseLinkSection(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_KEYWORD_LINKSECTION, &ok);
+ eatToken(p, TOKEN_KEYWORD_LINKSECTION, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseLinkSection cannot parse linksection\n");
@@ -156,7 +156,7 @@ static AstNodeIndex parseLinkSection(Parser* p) {
static AstNodeIndex parseCallconv(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_KEYWORD_CALLCONV, &ok);
+ eatToken(p, TOKEN_KEYWORD_CALLCONV, &ok);
if (!ok)
return null_node;
fprintf(stderr, "parseCallconv cannot parse callconv\n");
@@ -171,15 +171,15 @@ typedef struct {
} NodeContainerField;
static AstNodeIndex expectContainerField(Parser* p) {
- eatToken(p, TOKENIZER_TAG_KEYWORD_COMPTIME, NULL);
+ eatToken(p, TOKEN_KEYWORD_COMPTIME, NULL);
const AstTokenIndex main_token = p->tok_i;
- if (p->token_tags[p->tok_i] == TOKENIZER_TAG_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKENIZER_TAG_COLON)
+ if (p->token_tags[p->tok_i] == TOKEN_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKEN_COLON)
p->tok_i += 2;
const AstNodeIndex type_expr = parseTypeExpr(p);
const AstNodeIndex align_expr = parseByteAlign(p);
bool ok;
- eatToken(p, TOKENIZER_TAG_EQUAL, &ok);
+ eatToken(p, TOKEN_EQUAL, &ok);
if (ok) {
fprintf(stderr, "expectContainerField does not support expr\n");
exit(1);
@@ -225,27 +225,27 @@ static AstNodeIndex expectContainerField(Parser* p) {
static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
const TokenizerTag tok = p->token_tags[p->tok_i];
switch (tok) {
- case TOKENIZER_TAG_CHAR_LITERAL:
- case TOKENIZER_TAG_NUMBER_LITERAL:
- case TOKENIZER_TAG_KEYWORD_UNREACHABLE:
- case TOKENIZER_TAG_KEYWORD_ANYFRAME:
- case TOKENIZER_TAG_STRING_LITERAL:
- case TOKENIZER_TAG_BUILTIN:
- case TOKENIZER_TAG_KEYWORD_FN:
- case TOKENIZER_TAG_KEYWORD_IF:
- case TOKENIZER_TAG_KEYWORD_SWITCH:
- case TOKENIZER_TAG_KEYWORD_EXTERN:
- case TOKENIZER_TAG_KEYWORD_PACKED:
- case TOKENIZER_TAG_KEYWORD_STRUCT:
- case TOKENIZER_TAG_KEYWORD_OPAQUE:
- case TOKENIZER_TAG_KEYWORD_ENUM:
- case TOKENIZER_TAG_KEYWORD_UNION:
- case TOKENIZER_TAG_KEYWORD_COMPTIME:
- case TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE:
+ case TOKEN_CHAR_LITERAL:
+ case TOKEN_NUMBER_LITERAL:
+ case TOKEN_KEYWORD_UNREACHABLE:
+ case TOKEN_KEYWORD_ANYFRAME:
+ case TOKEN_STRING_LITERAL:
+ case TOKEN_BUILTIN:
+ case TOKEN_KEYWORD_FN:
+ case TOKEN_KEYWORD_IF:
+ case TOKEN_KEYWORD_SWITCH:
+ case TOKEN_KEYWORD_EXTERN:
+ case TOKEN_KEYWORD_PACKED:
+ case TOKEN_KEYWORD_STRUCT:
+ case TOKEN_KEYWORD_OPAQUE:
+ case TOKEN_KEYWORD_ENUM:
+ case TOKEN_KEYWORD_UNION:
+ case TOKEN_KEYWORD_COMPTIME:
+ case TOKEN_MULTILINE_STRING_LITERAL_LINE:
fprintf(stderr, "parsePrimaryTypeExpr does not support %s\n", tokenizerGetTagString(tok));
exit(1);
- case TOKENIZER_TAG_IDENTIFIER:
- if (p->token_tags[p->tok_i + 1] == TOKENIZER_TAG_COLON) {
+ case TOKEN_IDENTIFIER:
+ if (p->token_tags[p->tok_i + 1] == TOKEN_COLON) {
fprintf(stderr, "parsePrimaryTypeExpr does not support identifier followed by colon\n");
exit(1);
}
@@ -255,12 +255,12 @@ static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
.tag = AST_NODE_TAG_IDENTIFIER,
.main_token = nextToken(p),
.data = {} });
- case TOKENIZER_TAG_KEYWORD_INLINE:
- case TOKENIZER_TAG_KEYWORD_FOR:
- case TOKENIZER_TAG_KEYWORD_WHILE:
- case TOKENIZER_TAG_PERIOD:
- case TOKENIZER_TAG_KEYWORD_ERROR:
- case TOKENIZER_TAG_L_PAREN:
+ case TOKEN_KEYWORD_INLINE:
+ case TOKEN_KEYWORD_FOR:
+ case TOKEN_KEYWORD_WHILE:
+ case TOKEN_PERIOD:
+ case TOKEN_KEYWORD_ERROR:
+ case TOKEN_L_PAREN:
fprintf(stderr, "parsePrimaryTypeExpr does not support %s\n", tokenizerGetTagString(tok));
exit(1);
default:
@@ -271,10 +271,10 @@ static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
static AstNodeIndex parseSuffixOp(Parser* p) {
const TokenizerTag tok = p->token_tags[p->tok_i];
switch (tok) {
- case TOKENIZER_TAG_L_BRACKET:
- case TOKENIZER_TAG_PERIOD_ASTERISK:
- case TOKENIZER_TAG_INVALID_PERIODASTERISKS:
- case TOKENIZER_TAG_PERIOD:
+ case TOKEN_L_BRACKET:
+ case TOKEN_PERIOD_ASTERISK:
+ case TOKEN_INVALID_PERIODASTERISKS:
+ case TOKEN_PERIOD:
fprintf(stderr, "parseSuffixOp does not support %s\n", tokenizerGetTagString(tok));
exit(1);
default:
@@ -284,7 +284,7 @@ static AstNodeIndex parseSuffixOp(Parser* p) {
static AstNodeIndex parseSuffixExpr(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_KEYWORD_ASYNC, &ok);
+ eatToken(p, TOKEN_KEYWORD_ASYNC, &ok);
if (ok) {
fprintf(stderr, "async not supported\n");
exit(1);
@@ -300,13 +300,13 @@ static AstNodeIndex parseSuffixExpr(Parser* p) {
res = suffix_op;
continue;
}
- eatToken(p, TOKENIZER_TAG_L_PAREN, &ok);
+ eatToken(p, TOKEN_L_PAREN, &ok);
if (ok) {
fprintf(stderr, "parseSuffixExpr does not support expr with parens\n");
exit(1);
}
// TODO more work
- // const bool comma = p->token_tags[p->tok_i - 2] == TOKENIZER_TAG_COMMA;
+ // const bool comma = p->token_tags[p->tok_i - 2] == TOKEN_COMMA;
return res;
}
@@ -329,7 +329,7 @@ static AstNodeIndex parseErrorUnionExpr(Parser* p) {
if (suffix_expr == 0)
return null_node;
bool ok;
- const AstNodeIndex bang = eatToken(p, TOKENIZER_TAG_BANG, &ok);
+ const AstNodeIndex bang = eatToken(p, TOKEN_BANG, &ok);
if (!ok)
return suffix_expr;
return addNode(
@@ -347,11 +347,11 @@ static AstNodeIndex parseErrorUnionExpr(Parser* p) {
static AstNodeIndex parseTypeExpr(Parser* p) {
const AstNodeIndex tok = p->token_tags[p->tok_i];
switch (tok) {
- case TOKENIZER_TAG_QUESTION_MARK:
- case TOKENIZER_TAG_KEYWORD_ANYFRAME:
- case TOKENIZER_TAG_ASTERISK:
- case TOKENIZER_TAG_ASTERISK_ASTERISK:
- case TOKENIZER_TAG_L_BRACKET:
+ case TOKEN_QUESTION_MARK:
+ case TOKEN_KEYWORD_ANYFRAME:
+ case TOKEN_ASTERISK:
+ case TOKEN_ASTERISK_ASTERISK:
+ case TOKEN_L_BRACKET:
fprintf(stderr, "parseTypeExpr not supported for %s\n", tokenizerGetTagString(tok));
exit(1);
default:
@@ -362,13 +362,13 @@ static AstNodeIndex parseTypeExpr(Parser* p) {
static SmallSpan parseParamDeclList(Parser* p) {
// can only parse functions with no declarations
bool ok;
- AstTokenIndex got_token = eatToken(p, TOKENIZER_TAG_L_PAREN, &ok);
+ AstTokenIndex got_token = eatToken(p, TOKEN_L_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected (, got %s\n", tokenizerGetTagString(got_token));
exit(1);
}
- got_token = eatToken(p, TOKENIZER_TAG_R_PAREN, &ok);
+ got_token = eatToken(p, TOKEN_R_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected ), got %s\n", tokenizerGetTagString(got_token));
exit(1);
@@ -387,20 +387,20 @@ static uint32_t reserveNode(Parser* p, AstNodeTag tag) {
static AstNodeIndex parseFnProto(Parser* p) {
bool ok;
- AstNodeIndex fn_token = eatToken(p, TOKENIZER_TAG_KEYWORD_FN, &ok);
+ AstNodeIndex fn_token = eatToken(p, TOKEN_KEYWORD_FN, &ok);
if (!ok)
return null_node;
AstNodeIndex fn_proto_index = reserveNode(p, AST_NODE_TAG_FN_PROTO);
- eatToken(p, TOKENIZER_TAG_IDENTIFIER, NULL);
+ eatToken(p, TOKEN_IDENTIFIER, NULL);
SmallSpan params = parseParamDeclList(p);
const AstNodeIndex align_expr = parseByteAlign(p);
const AstNodeIndex addrspace_expr = parseAddrSpace(p);
const AstNodeIndex section_expr = parseLinkSection(p);
const AstNodeIndex callconv_expr = parseCallconv(p);
- eatToken(p, TOKENIZER_TAG_BANG, NULL);
+ eatToken(p, TOKEN_BANG, NULL);
const AstNodeIndex return_type_expr = parseTypeExpr(p);
@@ -425,7 +425,7 @@ static AstNodeIndex parseFnProto(Parser* p) {
}
static AstTokenIndex parseBlockLabel(Parser* p) {
- if (p->token_tags[p->tok_i] == TOKENIZER_TAG_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKENIZER_TAG_COLON) {
+ if (p->token_tags[p->tok_i] == TOKEN_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKEN_COLON) {
const AstTokenIndex identifier = p->tok_i;
p->tok_i += 2;
return identifier;
@@ -435,7 +435,7 @@ static AstTokenIndex parseBlockLabel(Parser* p) {
static AstNodeIndex parseForStatement(Parser* p) {
bool ok;
- const AstNodeIndex for_token = eatToken(p, TOKENIZER_TAG_KEYWORD_FOR, &ok);
+ const AstNodeIndex for_token = eatToken(p, TOKEN_KEYWORD_FOR, &ok);
if (!ok)
return null_node;
@@ -446,7 +446,7 @@ static AstNodeIndex parseForStatement(Parser* p) {
static AstNodeIndex parseWhileStatement(Parser* p) {
bool ok;
- const AstNodeIndex while_token = eatToken(p, TOKENIZER_TAG_KEYWORD_WHILE, &ok);
+ const AstNodeIndex while_token = eatToken(p, TOKEN_KEYWORD_WHILE, &ok);
if (!ok)
return null_node;
@@ -457,7 +457,7 @@ static AstNodeIndex parseWhileStatement(Parser* p) {
static AstNodeIndex parseLoopStatement(Parser* p) {
bool ok_inline_token;
- eatToken(p, TOKENIZER_TAG_KEYWORD_INLINE, &ok_inline_token);
+ eatToken(p, TOKEN_KEYWORD_INLINE, &ok_inline_token);
const AstNodeIndex for_statement = parseForStatement(p);
if (for_statement != 0)
@@ -492,21 +492,21 @@ static AstNodeIndex expectVarDeclExprStatement(Parser* p) {
static AstNodeIndex parseLabeledStatement(Parser*);
static AstNodeIndex expectStatement(Parser* p, bool allow_defer_var) {
bool ok;
- if (eatToken(p, TOKENIZER_TAG_KEYWORD_COMPTIME, &ok), ok) {
+ if (eatToken(p, TOKEN_KEYWORD_COMPTIME, &ok), ok) {
fprintf(stderr, "expectStatement: comptime keyword not supported\n");
exit(1);
}
const AstNodeIndex tok = p->token_tags[p->tok_i];
switch (tok) {
- case TOKENIZER_TAG_KEYWORD_NOSUSPEND:
- case TOKENIZER_TAG_KEYWORD_SUSPEND:
- case TOKENIZER_TAG_KEYWORD_DEFER:
- case TOKENIZER_TAG_KEYWORD_ERRDEFER:
- case TOKENIZER_TAG_KEYWORD_IF:
- case TOKENIZER_TAG_KEYWORD_ENUM:
- case TOKENIZER_TAG_KEYWORD_STRUCT:
- case TOKENIZER_TAG_KEYWORD_UNION:;
+ case TOKEN_KEYWORD_NOSUSPEND:
+ case TOKEN_KEYWORD_SUSPEND:
+ case TOKEN_KEYWORD_DEFER:
+ case TOKEN_KEYWORD_ERRDEFER:
+ case TOKEN_KEYWORD_IF:
+ case TOKEN_KEYWORD_ENUM:
+ case TOKEN_KEYWORD_STRUCT:
+ case TOKEN_KEYWORD_UNION:;
const char* tok_str = tokenizerGetTagString(tok);
fprintf(stderr, "expectStatement does not support keyword %s\n", tok_str);
exit(1);
@@ -532,7 +532,7 @@ static void cleanupScratch(CleanupScratch* c) { c->scratch->len = c->old_len; }
static AstNodeIndex parseBlock(Parser* p) {
bool ok;
- const AstNodeIndex lbrace = eatToken(p, TOKENIZER_TAG_L_BRACE, &ok);
+ const AstNodeIndex lbrace = eatToken(p, TOKEN_L_BRACE, &ok);
if (!ok)
return null_node;
@@ -542,7 +542,7 @@ static AstNodeIndex parseBlock(Parser* p) {
};
while (1) {
- if (p->token_tags[p->tok_i] == TOKENIZER_TAG_R_BRACE)
+ if (p->token_tags[p->tok_i] == TOKEN_R_BRACE)
break;
// "const AstNodeIndex statement" once tinycc supports typeof_unqual (C23)
@@ -551,8 +551,8 @@ static AstNodeIndex parseBlock(Parser* p) {
break;
SLICE_APPEND(AstNodeIndex, &p->scratch, statement);
}
- expectToken(p, TOKENIZER_TAG_R_BRACE, NULL);
- const bool semicolon = (p->token_tags[p->tok_i] - 2 == TOKENIZER_TAG_SEMICOLON);
+ expectToken(p, TOKEN_R_BRACE, NULL);
+ const bool semicolon = (p->token_tags[p->tok_i] - 2 == TOKEN_SEMICOLON);
const uint32_t statements_len = p->scratch.len - scratch_top.old_len;
switch (statements_len) {
@@ -626,9 +626,9 @@ static AstNodeIndex parseLabeledStatement(Parser* p) {
static AstNodeIndex parseVarDeclProto(Parser* p) {
bool ok;
- eatToken(p, TOKENIZER_TAG_KEYWORD_CONST, &ok);
+ eatToken(p, TOKEN_KEYWORD_CONST, &ok);
if (!ok) {
- eatToken(p, TOKENIZER_TAG_KEYWORD_VAR, &ok);
+ eatToken(p, TOKEN_KEYWORD_VAR, &ok);
if (!ok)
return null_node;
}
@@ -651,12 +651,12 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
AstTokenIndex extern_export_inline_token = nextToken(p);
switch (p->token_tags[extern_export_inline_token]) {
- case TOKENIZER_TAG_KEYWORD_EXTERN:
- eatToken(p, TOKENIZER_TAG_STRING_LITERAL, NULL);
+ case TOKEN_KEYWORD_EXTERN:
+ eatToken(p, TOKEN_STRING_LITERAL, NULL);
break;
- case TOKENIZER_TAG_KEYWORD_EXPORT:
- case TOKENIZER_TAG_KEYWORD_INLINE:
- case TOKENIZER_TAG_KEYWORD_NOINLINE:
+ case TOKEN_KEYWORD_EXPORT:
+ case TOKEN_KEYWORD_INLINE:
+ case TOKEN_KEYWORD_NOINLINE:
break;
default:
p->tok_i--;
@@ -665,10 +665,10 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
AstNodeIndex fn_proto = parseFnProto(p);
if (fn_proto != 0) {
switch (p->token_tags[p->tok_i]) {
- case TOKENIZER_TAG_SEMICOLON:
+ case TOKEN_SEMICOLON:
p->tok_i++;
return fn_proto;
- case TOKENIZER_TAG_L_BRACE:;
+ case TOKEN_L_BRACE:;
AstNodeIndex fn_decl_index = reserveNode(p, AST_NODE_TAG_FN_DECL);
AstNodeIndex body_block = parseBlock(p);
return setNode(
@@ -684,7 +684,7 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
}
}
- eatToken(p, TOKENIZER_TAG_KEYWORD_THREADLOCAL, NULL);
+ eatToken(p, TOKEN_KEYWORD_THREADLOCAL, NULL);
AstNodeIndex var_decl = parseGlobalVarDecl(p);
if (var_decl != 0) {
return var_decl;
@@ -704,46 +704,46 @@ void findNextContainerMember(Parser* p) {
switch (p->token_tags[tok]) {
// Any of these can start a new top level declaration
- case TOKENIZER_TAG_KEYWORD_TEST:
- case TOKENIZER_TAG_KEYWORD_COMPTIME:
- case TOKENIZER_TAG_KEYWORD_PUB:
- case TOKENIZER_TAG_KEYWORD_EXPORT:
- case TOKENIZER_TAG_KEYWORD_EXTERN:
- case TOKENIZER_TAG_KEYWORD_INLINE:
- case TOKENIZER_TAG_KEYWORD_NOINLINE:
- case TOKENIZER_TAG_KEYWORD_USINGNAMESPACE:
- case TOKENIZER_TAG_KEYWORD_THREADLOCAL:
- case TOKENIZER_TAG_KEYWORD_CONST:
- case TOKENIZER_TAG_KEYWORD_VAR:
- case TOKENIZER_TAG_KEYWORD_FN:
+ case TOKEN_KEYWORD_TEST:
+ case TOKEN_KEYWORD_COMPTIME:
+ case TOKEN_KEYWORD_PUB:
+ case TOKEN_KEYWORD_EXPORT:
+ case TOKEN_KEYWORD_EXTERN:
+ case TOKEN_KEYWORD_INLINE:
+ case TOKEN_KEYWORD_NOINLINE:
+ case TOKEN_KEYWORD_USINGNAMESPACE:
+ case TOKEN_KEYWORD_THREADLOCAL:
+ case TOKEN_KEYWORD_CONST:
+ case TOKEN_KEYWORD_VAR:
+ case TOKEN_KEYWORD_FN:
if (level == 0) {
p->tok_i--;
return;
}
break;
- case TOKENIZER_TAG_IDENTIFIER:
- if (p->token_tags[tok + 1] == TOKENIZER_TAG_COMMA && level == 0) {
+ case TOKEN_IDENTIFIER:
+ if (p->token_tags[tok + 1] == TOKEN_COMMA && level == 0) {
p->tok_i--;
return;
}
break;
- case TOKENIZER_TAG_COMMA:
- case TOKENIZER_TAG_SEMICOLON:
+ case TOKEN_COMMA:
+ case TOKEN_SEMICOLON:
// This decl was likely meant to end here
if (level == 0)
return;
break;
- case TOKENIZER_TAG_L_PAREN:
- case TOKENIZER_TAG_L_BRACKET:
- case TOKENIZER_TAG_L_BRACE:
+ case TOKEN_L_PAREN:
+ case TOKEN_L_BRACKET:
+ case TOKEN_L_BRACE:
level++;
break;
- case TOKENIZER_TAG_R_PAREN:
- case TOKENIZER_TAG_R_BRACKET:
+ case TOKEN_R_PAREN:
+ case TOKEN_R_BRACKET:
if (level != 0)
level--;
break;
- case TOKENIZER_TAG_R_BRACE:
+ case TOKEN_R_BRACE:
if (level == 0) {
// end of container, exit
p->tok_i--;
@@ -751,7 +751,7 @@ void findNextContainerMember(Parser* p) {
}
level--;
break;
- case TOKENIZER_TAG_EOF:
+ case TOKEN_EOF:
p->tok_i--;
return;
default:
@@ -766,7 +766,7 @@ static Members parseContainerMembers(Parser* p) {
.old_len = p->scratch.len,
};
bool ok;
- while (eatToken(p, TOKENIZER_TAG_CONTAINER_DOC_COMMENT, &ok), ok)
+ while (eatToken(p, TOKEN_CONTAINER_DOC_COMMENT, &ok), ok)
;
FieldState field_state = { .tag = FIELD_STATE_NONE };
@@ -775,13 +775,13 @@ static Members parseContainerMembers(Parser* p) {
while (1) {
eatDocComments(p);
switch (p->token_tags[p->tok_i]) {
- case TOKENIZER_TAG_KEYWORD_TEST:
- case TOKENIZER_TAG_KEYWORD_COMPTIME:
- case TOKENIZER_TAG_KEYWORD_USINGNAMESPACE:;
+ case TOKEN_KEYWORD_TEST:
+ case TOKEN_KEYWORD_COMPTIME:
+ case TOKEN_KEYWORD_USINGNAMESPACE:;
const char* str = tokenizerGetTagString(p->token_tags[p->tok_i]);
fprintf(stderr, "%s not implemented in parseContainerMembers\n", str);
exit(1);
- case TOKENIZER_TAG_KEYWORD_PUB: {
+ case TOKEN_KEYWORD_PUB: {
p->tok_i++;
AstNodeIndex top_level_decl = expectTopLevelDecl(p);
if (top_level_decl != 0) {
@@ -791,17 +791,17 @@ static Members parseContainerMembers(Parser* p) {
}
SLICE_APPEND(AstNodeIndex, &p->scratch, top_level_decl);
}
- trailing = p->token_tags[p->tok_i - 1] == TOKENIZER_TAG_SEMICOLON;
+ trailing = p->token_tags[p->tok_i - 1] == TOKEN_SEMICOLON;
break;
}
- case TOKENIZER_TAG_KEYWORD_CONST:
- case TOKENIZER_TAG_KEYWORD_VAR:
- case TOKENIZER_TAG_KEYWORD_THREADLOCAL:
- case TOKENIZER_TAG_KEYWORD_EXPORT:
- case TOKENIZER_TAG_KEYWORD_EXTERN:
- case TOKENIZER_TAG_KEYWORD_INLINE:
- case TOKENIZER_TAG_KEYWORD_NOINLINE:
- case TOKENIZER_TAG_KEYWORD_FN: {
+ case TOKEN_KEYWORD_CONST:
+ case TOKEN_KEYWORD_VAR:
+ case TOKEN_KEYWORD_THREADLOCAL:
+ case TOKEN_KEYWORD_EXPORT:
+ case TOKEN_KEYWORD_EXTERN:
+ case TOKEN_KEYWORD_INLINE:
+ case TOKEN_KEYWORD_NOINLINE:
+ case TOKEN_KEYWORD_FN: {
const AstNodeIndex top_level_decl = expectTopLevelDecl(p);
if (top_level_decl != 0) {
if (field_state.tag == FIELD_STATE_SEEN) {
@@ -810,11 +810,11 @@ static Members parseContainerMembers(Parser* p) {
}
SLICE_APPEND(AstNodeIndex, &p->scratch, top_level_decl);
}
- trailing = (p->token_tags[p->tok_i - 1] == TOKENIZER_TAG_SEMICOLON);
+ trailing = (p->token_tags[p->tok_i - 1] == TOKEN_SEMICOLON);
break;
}
- case TOKENIZER_TAG_EOF:
- case TOKENIZER_TAG_R_BRACE:
+ case TOKEN_EOF:
+ case TOKEN_R_BRACE:
goto break_loop;
default:;
// skip parseCStyleContainer
@@ -831,12 +831,12 @@ static Members parseContainerMembers(Parser* p) {
}
SLICE_APPEND(AstNodeIndex, &p->scratch, container_field);
switch (p->token_tags[p->tok_i]) {
- case TOKENIZER_TAG_COMMA:
+ case TOKEN_COMMA:
p->tok_i++;
trailing = true;
continue;
- case TOKENIZER_TAG_R_BRACE:
- case TOKENIZER_TAG_EOF:
+ case TOKEN_R_BRACE:
+ case TOKEN_EOF:
trailing = false;
goto break_loop;
default:;
diff --git a/tokenizer.c b/tokenizer.c
@@ -19,55 +19,55 @@ const char* tokenizerGetTagString(TokenizerTag tag) {
}
const KeywordMap keywords[] = {
- { "addrspace", TOKENIZER_TAG_KEYWORD_ADDRSPACE },
- { "align", TOKENIZER_TAG_KEYWORD_ALIGN },
- { "allowzero", TOKENIZER_TAG_KEYWORD_ALLOWZERO },
- { "and", TOKENIZER_TAG_KEYWORD_AND },
- { "anyframe", TOKENIZER_TAG_KEYWORD_ANYFRAME },
- { "anytype", TOKENIZER_TAG_KEYWORD_ANYTYPE },
- { "asm", TOKENIZER_TAG_KEYWORD_ASM },
- { "async", TOKENIZER_TAG_KEYWORD_ASYNC },
- { "await", TOKENIZER_TAG_KEYWORD_AWAIT },
- { "break", TOKENIZER_TAG_KEYWORD_BREAK },
- { "callconv", TOKENIZER_TAG_KEYWORD_CALLCONV },
- { "catch", TOKENIZER_TAG_KEYWORD_CATCH },
- { "comptime", TOKENIZER_TAG_KEYWORD_COMPTIME },
- { "const", TOKENIZER_TAG_KEYWORD_CONST },
- { "continue", TOKENIZER_TAG_KEYWORD_CONTINUE },
- { "defer", TOKENIZER_TAG_KEYWORD_DEFER },
- { "else", TOKENIZER_TAG_KEYWORD_ELSE },
- { "enum", TOKENIZER_TAG_KEYWORD_ENUM },
- { "errdefer", TOKENIZER_TAG_KEYWORD_ERRDEFER },
- { "error", TOKENIZER_TAG_KEYWORD_ERROR },
- { "export", TOKENIZER_TAG_KEYWORD_EXPORT },
- { "extern", TOKENIZER_TAG_KEYWORD_EXTERN },
- { "fn", TOKENIZER_TAG_KEYWORD_FN },
- { "for", TOKENIZER_TAG_KEYWORD_FOR },
- { "if", TOKENIZER_TAG_KEYWORD_IF },
- { "inline", TOKENIZER_TAG_KEYWORD_INLINE },
- { "linksection", TOKENIZER_TAG_KEYWORD_LINKSECTION },
- { "noalias", TOKENIZER_TAG_KEYWORD_NOALIAS },
- { "noinline", TOKENIZER_TAG_KEYWORD_NOINLINE },
- { "nosuspend", TOKENIZER_TAG_KEYWORD_NOSUSPEND },
- { "opaque", TOKENIZER_TAG_KEYWORD_OPAQUE },
- { "or", TOKENIZER_TAG_KEYWORD_OR },
- { "orelse", TOKENIZER_TAG_KEYWORD_ORELSE },
- { "packed", TOKENIZER_TAG_KEYWORD_PACKED },
- { "pub", TOKENIZER_TAG_KEYWORD_PUB },
- { "resume", TOKENIZER_TAG_KEYWORD_RESUME },
- { "return", TOKENIZER_TAG_KEYWORD_RETURN },
- { "struct", TOKENIZER_TAG_KEYWORD_STRUCT },
- { "suspend", TOKENIZER_TAG_KEYWORD_SUSPEND },
- { "switch", TOKENIZER_TAG_KEYWORD_SWITCH },
- { "test", TOKENIZER_TAG_KEYWORD_TEST },
- { "threadlocal", TOKENIZER_TAG_KEYWORD_THREADLOCAL },
- { "try", TOKENIZER_TAG_KEYWORD_TRY },
- { "union", TOKENIZER_TAG_KEYWORD_UNION },
- { "unreachable", TOKENIZER_TAG_KEYWORD_UNREACHABLE },
- { "usingnamespace", TOKENIZER_TAG_KEYWORD_USINGNAMESPACE },
- { "var", TOKENIZER_TAG_KEYWORD_VAR },
- { "volatile", TOKENIZER_TAG_KEYWORD_VOLATILE },
- { "while", TOKENIZER_TAG_KEYWORD_WHILE }
+ { "addrspace", TOKEN_KEYWORD_ADDRSPACE },
+ { "align", TOKEN_KEYWORD_ALIGN },
+ { "allowzero", TOKEN_KEYWORD_ALLOWZERO },
+ { "and", TOKEN_KEYWORD_AND },
+ { "anyframe", TOKEN_KEYWORD_ANYFRAME },
+ { "anytype", TOKEN_KEYWORD_ANYTYPE },
+ { "asm", TOKEN_KEYWORD_ASM },
+ { "async", TOKEN_KEYWORD_ASYNC },
+ { "await", TOKEN_KEYWORD_AWAIT },
+ { "break", TOKEN_KEYWORD_BREAK },
+ { "callconv", TOKEN_KEYWORD_CALLCONV },
+ { "catch", TOKEN_KEYWORD_CATCH },
+ { "comptime", TOKEN_KEYWORD_COMPTIME },
+ { "const", TOKEN_KEYWORD_CONST },
+ { "continue", TOKEN_KEYWORD_CONTINUE },
+ { "defer", TOKEN_KEYWORD_DEFER },
+ { "else", TOKEN_KEYWORD_ELSE },
+ { "enum", TOKEN_KEYWORD_ENUM },
+ { "errdefer", TOKEN_KEYWORD_ERRDEFER },
+ { "error", TOKEN_KEYWORD_ERROR },
+ { "export", TOKEN_KEYWORD_EXPORT },
+ { "extern", TOKEN_KEYWORD_EXTERN },
+ { "fn", TOKEN_KEYWORD_FN },
+ { "for", TOKEN_KEYWORD_FOR },
+ { "if", TOKEN_KEYWORD_IF },
+ { "inline", TOKEN_KEYWORD_INLINE },
+ { "linksection", TOKEN_KEYWORD_LINKSECTION },
+ { "noalias", TOKEN_KEYWORD_NOALIAS },
+ { "noinline", TOKEN_KEYWORD_NOINLINE },
+ { "nosuspend", TOKEN_KEYWORD_NOSUSPEND },
+ { "opaque", TOKEN_KEYWORD_OPAQUE },
+ { "or", TOKEN_KEYWORD_OR },
+ { "orelse", TOKEN_KEYWORD_ORELSE },
+ { "packed", TOKEN_KEYWORD_PACKED },
+ { "pub", TOKEN_KEYWORD_PUB },
+ { "resume", TOKEN_KEYWORD_RESUME },
+ { "return", TOKEN_KEYWORD_RETURN },
+ { "struct", TOKEN_KEYWORD_STRUCT },
+ { "suspend", TOKEN_KEYWORD_SUSPEND },
+ { "switch", TOKEN_KEYWORD_SWITCH },
+ { "test", TOKEN_KEYWORD_TEST },
+ { "threadlocal", TOKEN_KEYWORD_THREADLOCAL },
+ { "try", TOKEN_KEYWORD_TRY },
+ { "union", TOKEN_KEYWORD_UNION },
+ { "unreachable", TOKEN_KEYWORD_UNREACHABLE },
+ { "usingnamespace", TOKEN_KEYWORD_USINGNAMESPACE },
+ { "var", TOKEN_KEYWORD_VAR },
+ { "volatile", TOKEN_KEYWORD_VOLATILE },
+ { "while", TOKEN_KEYWORD_WHILE }
};
// TODO binary search
@@ -80,13 +80,13 @@ static TokenizerTag getKeyword(const char* bytes, const uint32_t len) {
if (len == klen) {
return keywords[i].tag;
} else {
- return TOKENIZER_TAG_INVALID;
+ return TOKEN_INVALID;
}
} else if (cmp < 0) {
- return TOKENIZER_TAG_INVALID;
+ return TOKEN_INVALID;
}
}
- return TOKENIZER_TAG_INVALID;
+ return TOKEN_INVALID;
}
Tokenizer tokenizerInit(const char* buffer, const uint32_t len) {
@@ -99,7 +99,7 @@ Tokenizer tokenizerInit(const char* buffer, const uint32_t len) {
TokenizerToken tokenizerNext(Tokenizer* self) {
TokenizerToken result = (TokenizerToken) {
- .tag = TOKENIZER_TAG_INVALID,
+ .tag = TOKEN_INVALID,
.loc = {
.start = 0,
},
@@ -114,7 +114,7 @@ state:
case 0:
if (self->index == self->buffer_len) {
return (TokenizerToken) {
- .tag = TOKENIZER_TAG_EOF,
+ .tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index,
@@ -132,17 +132,17 @@ state:
result.loc.start = self->index;
goto state;
case '"':
- result.tag = TOKENIZER_TAG_STRING_LITERAL;
+ result.tag = TOKEN_STRING_LITERAL;
state = TOKENIZER_STATE_STRING_LITERAL;
goto state;
case '\'':
- result.tag = TOKENIZER_TAG_CHAR_LITERAL;
+ result.tag = TOKEN_CHAR_LITERAL;
state = TOKENIZER_STATE_CHAR_LITERAL;
goto state;
case 'a' ... 'z':
case 'A' ... 'Z':
case '_':
- result.tag = TOKENIZER_TAG_IDENTIFIER;
+ result.tag = TOKEN_IDENTIFIER;
state = TOKENIZER_STATE_IDENTIFIER;
goto state;
case '@':
@@ -158,35 +158,35 @@ state:
state = TOKENIZER_STATE_PIPE;
goto state;
case '(':
- result.tag = TOKENIZER_TAG_L_PAREN;
+ result.tag = TOKEN_L_PAREN;
self->index++;
break;
case ')':
- result.tag = TOKENIZER_TAG_R_PAREN;
+ result.tag = TOKEN_R_PAREN;
self->index++;
break;
case '[':
- result.tag = TOKENIZER_TAG_L_BRACKET;
+ result.tag = TOKEN_L_BRACKET;
self->index++;
break;
case ']':
- result.tag = TOKENIZER_TAG_R_BRACKET;
+ result.tag = TOKEN_R_BRACKET;
self->index++;
break;
case ';':
- result.tag = TOKENIZER_TAG_SEMICOLON;
+ result.tag = TOKEN_SEMICOLON;
self->index++;
break;
case ',':
- result.tag = TOKENIZER_TAG_COMMA;
+ result.tag = TOKEN_COMMA;
self->index++;
break;
case '?':
- result.tag = TOKENIZER_TAG_QUESTION_MARK;
+ result.tag = TOKEN_QUESTION_MARK;
self->index++;
break;
case ':':
- result.tag = TOKENIZER_TAG_COLON;
+ result.tag = TOKEN_COLON;
self->index++;
break;
case '%':
@@ -208,19 +208,19 @@ state:
state = TOKENIZER_STATE_CARET;
goto state;
case '\\':
- result.tag = TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE;
+ result.tag = TOKEN_MULTILINE_STRING_LITERAL_LINE;
state = TOKENIZER_STATE_BACKSLASH;
goto state;
case '{':
- result.tag = TOKENIZER_TAG_L_BRACE;
+ result.tag = TOKEN_L_BRACE;
self->index++;
break;
case '}':
- result.tag = TOKENIZER_TAG_R_BRACE;
+ result.tag = TOKEN_R_BRACE;
self->index++;
break;
case '~':
- result.tag = TOKENIZER_TAG_TILDE;
+ result.tag = TOKEN_TILDE;
self->index++;
break;
case '.':
@@ -236,7 +236,7 @@ state:
state = TOKENIZER_STATE_AMPERSAND;
goto state;
case '0' ... '9':
- result.tag = TOKENIZER_TAG_NUMBER_LITERAL;
+ result.tag = TOKEN_NUMBER_LITERAL;
self->index++;
state = TOKENIZER_STATE_INT;
goto state;
@@ -251,7 +251,7 @@ state:
switch (self->buffer[self->index]) {
case 0:
if (self->index == self->buffer_len) {
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
@@ -273,14 +273,14 @@ state:
switch (self->buffer[self->index]) {
case 0:
if (self->index == self->buffer_len) {
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
}
break;
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_INVALID;
@@ -293,16 +293,16 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
case '"':
- result.tag = TOKENIZER_TAG_IDENTIFIER;
+ result.tag = TOKEN_IDENTIFIER;
state = TOKENIZER_STATE_STRING_LITERAL;
goto state;
case 'a' ... 'z':
case 'A' ... 'Z':
case '_':
- result.tag = TOKENIZER_TAG_BUILTIN;
+ result.tag = TOKEN_BUILTIN;
state = TOKENIZER_STATE_BUILTIN;
goto state;
default:
@@ -315,11 +315,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_AMPERSAND_EQUAL;
+ result.tag = TOKEN_AMPERSAND_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_AMPERSAND;
+ result.tag = TOKEN_AMPERSAND;
break;
}
break;
@@ -328,11 +328,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_ASTERISK_EQUAL;
+ result.tag = TOKEN_ASTERISK_EQUAL;
self->index++;
break;
case '*':
- result.tag = TOKENIZER_TAG_ASTERISK_ASTERISK;
+ result.tag = TOKEN_ASTERISK_ASTERISK;
self->index++;
break;
case '%':
@@ -342,7 +342,7 @@ state:
state = TOKENIZER_STATE_ASTERISK_PIPE;
goto state;
default:
- result.tag = TOKENIZER_TAG_ASTERISK;
+ result.tag = TOKEN_ASTERISK;
break;
}
break;
@@ -351,11 +351,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL;
+ result.tag = TOKEN_ASTERISK_PERCENT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ASTERISK_PERCENT;
+ result.tag = TOKEN_ASTERISK_PERCENT;
break;
}
break;
@@ -364,11 +364,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_ASTERISK_PIPE_EQUAL;
+ result.tag = TOKEN_ASTERISK_PIPE_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ASTERISK_PIPE;
+ result.tag = TOKEN_ASTERISK_PIPE;
break;
}
break;
@@ -377,11 +377,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_PERCENT_EQUAL;
+ result.tag = TOKEN_PERCENT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_PERCENT;
+ result.tag = TOKEN_PERCENT;
break;
}
break;
@@ -390,11 +390,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_PLUS_EQUAL;
+ result.tag = TOKEN_PLUS_EQUAL;
self->index++;
break;
case '+':
- result.tag = TOKENIZER_TAG_PLUS_PLUS;
+ result.tag = TOKEN_PLUS_PLUS;
self->index++;
break;
case '%':
@@ -404,7 +404,7 @@ state:
state = TOKENIZER_STATE_PLUS_PIPE;
goto state;
default:
- result.tag = TOKENIZER_TAG_PLUS;
+ result.tag = TOKEN_PLUS;
break;
}
break;
@@ -413,11 +413,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_PLUS_PERCENT_EQUAL;
+ result.tag = TOKEN_PLUS_PERCENT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_PLUS_PERCENT;
+ result.tag = TOKEN_PLUS_PERCENT;
break;
}
break;
@@ -426,11 +426,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_PLUS_PIPE_EQUAL;
+ result.tag = TOKEN_PLUS_PIPE_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_PLUS_PIPE;
+ result.tag = TOKEN_PLUS_PIPE;
break;
}
break;
@@ -439,11 +439,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_CARET_EQUAL;
+ result.tag = TOKEN_CARET_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_CARET;
+ result.tag = TOKEN_CARET;
break;
}
break;
@@ -461,7 +461,7 @@ state:
const char* start = self->buffer + result.loc.start;
uint32_t len = self->index - result.loc.start;
TokenizerTag tag = getKeyword(start, len);
- if (tag != TOKENIZER_TAG_INVALID) {
+ if (tag != TOKEN_INVALID) {
result.tag = tag;
}
}
@@ -484,13 +484,13 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case 0:
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_MULTILINE_STRING_LITERAL_LINE;
goto state;
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_INVALID;
@@ -506,11 +506,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
}
break;
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_STRING_LITERAL_BACKSLASH;
@@ -534,7 +534,7 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_STRING_LITERAL;
@@ -550,11 +550,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
}
break;
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_CHAR_LITERAL_BACKSLASH;
@@ -581,11 +581,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
}
break;
case '\n':
- result.tag = TOKENIZER_TAG_INVALID;
+ result.tag = TOKEN_INVALID;
break;
case 0x01 ... 0x09:
case 0x0b ... 0x1f:
@@ -631,11 +631,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_BANG_EQUAL;
+ result.tag = TOKEN_BANG_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_BANG;
+ result.tag = TOKEN_BANG;
break;
}
break;
@@ -644,15 +644,15 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_PIPE_EQUAL;
+ result.tag = TOKEN_PIPE_EQUAL;
self->index++;
break;
case '|':
- result.tag = TOKENIZER_TAG_PIPE_PIPE;
+ result.tag = TOKEN_PIPE_PIPE;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_PIPE;
+ result.tag = TOKEN_PIPE;
break;
}
break;
@@ -661,15 +661,15 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_EQUAL_EQUAL;
+ result.tag = TOKEN_EQUAL_EQUAL;
self->index++;
break;
case '>':
- result.tag = TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT;
+ result.tag = TOKEN_EQUAL_ANGLE_BRACKET_RIGHT;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_EQUAL;
+ result.tag = TOKEN_EQUAL;
break;
}
break;
@@ -678,11 +678,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '>':
- result.tag = TOKENIZER_TAG_ARROW;
+ result.tag = TOKEN_ARROW;
self->index++;
break;
case '=':
- result.tag = TOKENIZER_TAG_MINUS_EQUAL;
+ result.tag = TOKEN_MINUS_EQUAL;
self->index++;
break;
case '%':
@@ -692,7 +692,7 @@ state:
state = TOKENIZER_STATE_MINUS_PIPE;
goto state;
default:
- result.tag = TOKENIZER_TAG_MINUS;
+ result.tag = TOKEN_MINUS;
break;
}
break;
@@ -701,11 +701,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_MINUS_PERCENT_EQUAL;
+ result.tag = TOKEN_MINUS_PERCENT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_MINUS_PERCENT;
+ result.tag = TOKEN_MINUS_PERCENT;
break;
}
break;
@@ -714,11 +714,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_MINUS_PIPE_EQUAL;
+ result.tag = TOKEN_MINUS_PIPE_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_MINUS_PIPE;
+ result.tag = TOKEN_MINUS_PIPE;
break;
}
break;
@@ -730,11 +730,11 @@ state:
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
goto state;
case '=':
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL;
+ result.tag = TOKEN_ANGLE_BRACKET_LEFT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_LEFT;
+ result.tag = TOKEN_ANGLE_BRACKET_LEFT;
break;
}
break;
@@ -743,14 +743,14 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL;
+ result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL;
self->index++;
break;
case '|':
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
goto state;
default:
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
+ result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
break;
}
break;
@@ -759,11 +759,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL;
+ result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
+ result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
break;
}
break;
@@ -775,11 +775,11 @@ state:
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
goto state;
case '=':
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL;
+ result.tag = TOKEN_ANGLE_BRACKET_RIGHT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_RIGHT;
+ result.tag = TOKEN_ANGLE_BRACKET_RIGHT;
break;
}
break;
@@ -788,11 +788,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL;
+ result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
+ result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
break;
}
break;
@@ -807,7 +807,7 @@ state:
state = TOKENIZER_STATE_PERIOD_ASTERISK;
goto state;
default:
- result.tag = TOKENIZER_TAG_PERIOD;
+ result.tag = TOKEN_PERIOD;
break;
}
break;
@@ -816,11 +816,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '.':
- result.tag = TOKENIZER_TAG_ELLIPSIS3;
+ result.tag = TOKEN_ELLIPSIS3;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_ELLIPSIS2;
+ result.tag = TOKEN_ELLIPSIS2;
break;
}
break;
@@ -829,10 +829,10 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '*':
- result.tag = TOKENIZER_TAG_INVALID_PERIODASTERISKS;
+ result.tag = TOKEN_INVALID_PERIODASTERISKS;
break;
default:
- result.tag = TOKENIZER_TAG_PERIOD_ASTERISK;
+ result.tag = TOKEN_PERIOD_ASTERISK;
break;
}
break;
@@ -844,11 +844,11 @@ state:
state = TOKENIZER_STATE_LINE_COMMENT_START;
goto state;
case '=':
- result.tag = TOKENIZER_TAG_SLASH_EQUAL;
+ result.tag = TOKEN_SLASH_EQUAL;
self->index++;
break;
default:
- result.tag = TOKENIZER_TAG_SLASH;
+ result.tag = TOKEN_SLASH;
break;
}
break;
@@ -862,7 +862,7 @@ state:
goto state;
} else {
return (TokenizerToken) {
- .tag = TOKENIZER_TAG_EOF,
+ .tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index }
@@ -870,7 +870,7 @@ state:
}
break;
case '!':
- result.tag = TOKENIZER_TAG_CONTAINER_DOC_COMMENT;
+ result.tag = TOKEN_CONTAINER_DOC_COMMENT;
state = TOKENIZER_STATE_DOC_COMMENT;
goto state;
case '\n':
@@ -901,11 +901,11 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
- result.tag = TOKENIZER_TAG_DOC_COMMENT;
+ result.tag = TOKEN_DOC_COMMENT;
break;
case '\r':
if (self->buffer[self->index + 1] == '\n') {
- result.tag = TOKENIZER_TAG_DOC_COMMENT;
+ result.tag = TOKEN_DOC_COMMENT;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
@@ -921,7 +921,7 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
default:
- result.tag = TOKENIZER_TAG_DOC_COMMENT;
+ result.tag = TOKEN_DOC_COMMENT;
state = TOKENIZER_STATE_DOC_COMMENT;
goto state;
}
@@ -936,7 +936,7 @@ state:
goto state;
} else {
return (TokenizerToken) {
- .tag = TOKENIZER_TAG_EOF,
+ .tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index }
diff --git a/tokenizer.h b/tokenizer.h
@@ -5,128 +5,128 @@
#include <stdint.h>
#define TOKENIZER_FOREACH_TAG_ENUM(TAG) \
- TAG(TOKENIZER_TAG_INVALID) \
- TAG(TOKENIZER_TAG_INVALID_PERIODASTERISKS) \
- TAG(TOKENIZER_TAG_IDENTIFIER) \
- TAG(TOKENIZER_TAG_STRING_LITERAL) \
- TAG(TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE) \
- TAG(TOKENIZER_TAG_CHAR_LITERAL) \
- TAG(TOKENIZER_TAG_EOF) \
- TAG(TOKENIZER_TAG_BUILTIN) \
- TAG(TOKENIZER_TAG_BANG) \
- TAG(TOKENIZER_TAG_PIPE) \
- TAG(TOKENIZER_TAG_PIPE_PIPE) \
- TAG(TOKENIZER_TAG_PIPE_EQUAL) \
- TAG(TOKENIZER_TAG_EQUAL) \
- TAG(TOKENIZER_TAG_EQUAL_EQUAL) \
- TAG(TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT) \
- TAG(TOKENIZER_TAG_BANG_EQUAL) \
- TAG(TOKENIZER_TAG_L_PAREN) \
- TAG(TOKENIZER_TAG_R_PAREN) \
- TAG(TOKENIZER_TAG_SEMICOLON) \
- TAG(TOKENIZER_TAG_PERCENT) \
- TAG(TOKENIZER_TAG_PERCENT_EQUAL) \
- TAG(TOKENIZER_TAG_L_BRACE) \
- TAG(TOKENIZER_TAG_R_BRACE) \
- TAG(TOKENIZER_TAG_L_BRACKET) \
- TAG(TOKENIZER_TAG_R_BRACKET) \
- TAG(TOKENIZER_TAG_PERIOD) \
- TAG(TOKENIZER_TAG_PERIOD_ASTERISK) \
- TAG(TOKENIZER_TAG_ELLIPSIS2) \
- TAG(TOKENIZER_TAG_ELLIPSIS3) \
- TAG(TOKENIZER_TAG_CARET) \
- TAG(TOKENIZER_TAG_CARET_EQUAL) \
- TAG(TOKENIZER_TAG_PLUS) \
- TAG(TOKENIZER_TAG_PLUS_PLUS) \
- TAG(TOKENIZER_TAG_PLUS_EQUAL) \
- TAG(TOKENIZER_TAG_PLUS_PERCENT) \
- TAG(TOKENIZER_TAG_PLUS_PERCENT_EQUAL) \
- TAG(TOKENIZER_TAG_PLUS_PIPE) \
- TAG(TOKENIZER_TAG_PLUS_PIPE_EQUAL) \
- TAG(TOKENIZER_TAG_MINUS) \
- TAG(TOKENIZER_TAG_MINUS_EQUAL) \
- TAG(TOKENIZER_TAG_MINUS_PERCENT) \
- TAG(TOKENIZER_TAG_MINUS_PERCENT_EQUAL) \
- TAG(TOKENIZER_TAG_MINUS_PIPE) \
- TAG(TOKENIZER_TAG_MINUS_PIPE_EQUAL) \
- TAG(TOKENIZER_TAG_ASTERISK) \
- TAG(TOKENIZER_TAG_ASTERISK_EQUAL) \
- TAG(TOKENIZER_TAG_ASTERISK_ASTERISK) \
- TAG(TOKENIZER_TAG_ASTERISK_PERCENT) \
- TAG(TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL) \
- TAG(TOKENIZER_TAG_ASTERISK_PIPE) \
- TAG(TOKENIZER_TAG_ASTERISK_PIPE_EQUAL) \
- TAG(TOKENIZER_TAG_ARROW) \
- TAG(TOKENIZER_TAG_COLON) \
- TAG(TOKENIZER_TAG_SLASH) \
- TAG(TOKENIZER_TAG_SLASH_EQUAL) \
- TAG(TOKENIZER_TAG_COMMA) \
- TAG(TOKENIZER_TAG_AMPERSAND) \
- TAG(TOKENIZER_TAG_AMPERSAND_EQUAL) \
- TAG(TOKENIZER_TAG_QUESTION_MARK) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_LEFT) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_RIGHT) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT) \
- TAG(TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL) \
- TAG(TOKENIZER_TAG_TILDE) \
- TAG(TOKENIZER_TAG_NUMBER_LITERAL) \
- TAG(TOKENIZER_TAG_DOC_COMMENT) \
- TAG(TOKENIZER_TAG_CONTAINER_DOC_COMMENT) \
- TAG(TOKENIZER_TAG_KEYWORD_ADDRSPACE) \
- TAG(TOKENIZER_TAG_KEYWORD_ALIGN) \
- TAG(TOKENIZER_TAG_KEYWORD_ALLOWZERO) \
- TAG(TOKENIZER_TAG_KEYWORD_AND) \
- TAG(TOKENIZER_TAG_KEYWORD_ANYFRAME) \
- TAG(TOKENIZER_TAG_KEYWORD_ANYTYPE) \
- TAG(TOKENIZER_TAG_KEYWORD_ASM) \
- TAG(TOKENIZER_TAG_KEYWORD_ASYNC) \
- TAG(TOKENIZER_TAG_KEYWORD_AWAIT) \
- TAG(TOKENIZER_TAG_KEYWORD_BREAK) \
- TAG(TOKENIZER_TAG_KEYWORD_CALLCONV) \
- TAG(TOKENIZER_TAG_KEYWORD_CATCH) \
- TAG(TOKENIZER_TAG_KEYWORD_COMPTIME) \
- TAG(TOKENIZER_TAG_KEYWORD_CONST) \
- TAG(TOKENIZER_TAG_KEYWORD_CONTINUE) \
- TAG(TOKENIZER_TAG_KEYWORD_DEFER) \
- TAG(TOKENIZER_TAG_KEYWORD_ELSE) \
- TAG(TOKENIZER_TAG_KEYWORD_ENUM) \
- TAG(TOKENIZER_TAG_KEYWORD_ERRDEFER) \
- TAG(TOKENIZER_TAG_KEYWORD_ERROR) \
- TAG(TOKENIZER_TAG_KEYWORD_EXPORT) \
- TAG(TOKENIZER_TAG_KEYWORD_EXTERN) \
- TAG(TOKENIZER_TAG_KEYWORD_FN) \
- TAG(TOKENIZER_TAG_KEYWORD_FOR) \
- TAG(TOKENIZER_TAG_KEYWORD_IF) \
- TAG(TOKENIZER_TAG_KEYWORD_INLINE) \
- TAG(TOKENIZER_TAG_KEYWORD_NOALIAS) \
- TAG(TOKENIZER_TAG_KEYWORD_NOINLINE) \
- TAG(TOKENIZER_TAG_KEYWORD_NOSUSPEND) \
- TAG(TOKENIZER_TAG_KEYWORD_OPAQUE) \
- TAG(TOKENIZER_TAG_KEYWORD_OR) \
- TAG(TOKENIZER_TAG_KEYWORD_ORELSE) \
- TAG(TOKENIZER_TAG_KEYWORD_PACKED) \
- TAG(TOKENIZER_TAG_KEYWORD_PUB) \
- TAG(TOKENIZER_TAG_KEYWORD_RESUME) \
- TAG(TOKENIZER_TAG_KEYWORD_RETURN) \
- TAG(TOKENIZER_TAG_KEYWORD_LINKSECTION) \
- TAG(TOKENIZER_TAG_KEYWORD_STRUCT) \
- TAG(TOKENIZER_TAG_KEYWORD_SUSPEND) \
- TAG(TOKENIZER_TAG_KEYWORD_SWITCH) \
- TAG(TOKENIZER_TAG_KEYWORD_TEST) \
- TAG(TOKENIZER_TAG_KEYWORD_THREADLOCAL) \
- TAG(TOKENIZER_TAG_KEYWORD_TRY) \
- TAG(TOKENIZER_TAG_KEYWORD_UNION) \
- TAG(TOKENIZER_TAG_KEYWORD_UNREACHABLE) \
- TAG(TOKENIZER_TAG_KEYWORD_USINGNAMESPACE) \
- TAG(TOKENIZER_TAG_KEYWORD_VAR) \
- TAG(TOKENIZER_TAG_KEYWORD_VOLATILE) \
- TAG(TOKENIZER_TAG_KEYWORD_WHILE)
+ TAG(TOKEN_INVALID) \
+ TAG(TOKEN_INVALID_PERIODASTERISKS) \
+ TAG(TOKEN_IDENTIFIER) \
+ TAG(TOKEN_STRING_LITERAL) \
+ TAG(TOKEN_MULTILINE_STRING_LITERAL_LINE) \
+ TAG(TOKEN_CHAR_LITERAL) \
+ TAG(TOKEN_EOF) \
+ TAG(TOKEN_BUILTIN) \
+ TAG(TOKEN_BANG) \
+ TAG(TOKEN_PIPE) \
+ TAG(TOKEN_PIPE_PIPE) \
+ TAG(TOKEN_PIPE_EQUAL) \
+ TAG(TOKEN_EQUAL) \
+ TAG(TOKEN_EQUAL_EQUAL) \
+ TAG(TOKEN_EQUAL_ANGLE_BRACKET_RIGHT) \
+ TAG(TOKEN_BANG_EQUAL) \
+ TAG(TOKEN_L_PAREN) \
+ TAG(TOKEN_R_PAREN) \
+ TAG(TOKEN_SEMICOLON) \
+ TAG(TOKEN_PERCENT) \
+ TAG(TOKEN_PERCENT_EQUAL) \
+ TAG(TOKEN_L_BRACE) \
+ TAG(TOKEN_R_BRACE) \
+ TAG(TOKEN_L_BRACKET) \
+ TAG(TOKEN_R_BRACKET) \
+ TAG(TOKEN_PERIOD) \
+ TAG(TOKEN_PERIOD_ASTERISK) \
+ TAG(TOKEN_ELLIPSIS2) \
+ TAG(TOKEN_ELLIPSIS3) \
+ TAG(TOKEN_CARET) \
+ TAG(TOKEN_CARET_EQUAL) \
+ TAG(TOKEN_PLUS) \
+ TAG(TOKEN_PLUS_PLUS) \
+ TAG(TOKEN_PLUS_EQUAL) \
+ TAG(TOKEN_PLUS_PERCENT) \
+ TAG(TOKEN_PLUS_PERCENT_EQUAL) \
+ TAG(TOKEN_PLUS_PIPE) \
+ TAG(TOKEN_PLUS_PIPE_EQUAL) \
+ TAG(TOKEN_MINUS) \
+ TAG(TOKEN_MINUS_EQUAL) \
+ TAG(TOKEN_MINUS_PERCENT) \
+ TAG(TOKEN_MINUS_PERCENT_EQUAL) \
+ TAG(TOKEN_MINUS_PIPE) \
+ TAG(TOKEN_MINUS_PIPE_EQUAL) \
+ TAG(TOKEN_ASTERISK) \
+ TAG(TOKEN_ASTERISK_EQUAL) \
+ TAG(TOKEN_ASTERISK_ASTERISK) \
+ TAG(TOKEN_ASTERISK_PERCENT) \
+ TAG(TOKEN_ASTERISK_PERCENT_EQUAL) \
+ TAG(TOKEN_ASTERISK_PIPE) \
+ TAG(TOKEN_ASTERISK_PIPE_EQUAL) \
+ TAG(TOKEN_ARROW) \
+ TAG(TOKEN_COLON) \
+ TAG(TOKEN_SLASH) \
+ TAG(TOKEN_SLASH_EQUAL) \
+ TAG(TOKEN_COMMA) \
+ TAG(TOKEN_AMPERSAND) \
+ TAG(TOKEN_AMPERSAND_EQUAL) \
+ TAG(TOKEN_QUESTION_MARK) \
+ TAG(TOKEN_ANGLE_BRACKET_LEFT) \
+ TAG(TOKEN_ANGLE_BRACKET_LEFT_EQUAL) \
+ TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT) \
+ TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL) \
+ TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE) \
+ TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL) \
+ TAG(TOKEN_ANGLE_BRACKET_RIGHT) \
+ TAG(TOKEN_ANGLE_BRACKET_RIGHT_EQUAL) \
+ TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT) \
+ TAG(TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL) \
+ TAG(TOKEN_TILDE) \
+ TAG(TOKEN_NUMBER_LITERAL) \
+ TAG(TOKEN_DOC_COMMENT) \
+ TAG(TOKEN_CONTAINER_DOC_COMMENT) \
+ TAG(TOKEN_KEYWORD_ADDRSPACE) \
+ TAG(TOKEN_KEYWORD_ALIGN) \
+ TAG(TOKEN_KEYWORD_ALLOWZERO) \
+ TAG(TOKEN_KEYWORD_AND) \
+ TAG(TOKEN_KEYWORD_ANYFRAME) \
+ TAG(TOKEN_KEYWORD_ANYTYPE) \
+ TAG(TOKEN_KEYWORD_ASM) \
+ TAG(TOKEN_KEYWORD_ASYNC) \
+ TAG(TOKEN_KEYWORD_AWAIT) \
+ TAG(TOKEN_KEYWORD_BREAK) \
+ TAG(TOKEN_KEYWORD_CALLCONV) \
+ TAG(TOKEN_KEYWORD_CATCH) \
+ TAG(TOKEN_KEYWORD_COMPTIME) \
+ TAG(TOKEN_KEYWORD_CONST) \
+ TAG(TOKEN_KEYWORD_CONTINUE) \
+ TAG(TOKEN_KEYWORD_DEFER) \
+ TAG(TOKEN_KEYWORD_ELSE) \
+ TAG(TOKEN_KEYWORD_ENUM) \
+ TAG(TOKEN_KEYWORD_ERRDEFER) \
+ TAG(TOKEN_KEYWORD_ERROR) \
+ TAG(TOKEN_KEYWORD_EXPORT) \
+ TAG(TOKEN_KEYWORD_EXTERN) \
+ TAG(TOKEN_KEYWORD_FN) \
+ TAG(TOKEN_KEYWORD_FOR) \
+ TAG(TOKEN_KEYWORD_IF) \
+ TAG(TOKEN_KEYWORD_INLINE) \
+ TAG(TOKEN_KEYWORD_NOALIAS) \
+ TAG(TOKEN_KEYWORD_NOINLINE) \
+ TAG(TOKEN_KEYWORD_NOSUSPEND) \
+ TAG(TOKEN_KEYWORD_OPAQUE) \
+ TAG(TOKEN_KEYWORD_OR) \
+ TAG(TOKEN_KEYWORD_ORELSE) \
+ TAG(TOKEN_KEYWORD_PACKED) \
+ TAG(TOKEN_KEYWORD_PUB) \
+ TAG(TOKEN_KEYWORD_RESUME) \
+ TAG(TOKEN_KEYWORD_RETURN) \
+ TAG(TOKEN_KEYWORD_LINKSECTION) \
+ TAG(TOKEN_KEYWORD_STRUCT) \
+ TAG(TOKEN_KEYWORD_SUSPEND) \
+ TAG(TOKEN_KEYWORD_SWITCH) \
+ TAG(TOKEN_KEYWORD_TEST) \
+ TAG(TOKEN_KEYWORD_THREADLOCAL) \
+ TAG(TOKEN_KEYWORD_TRY) \
+ TAG(TOKEN_KEYWORD_UNION) \
+ TAG(TOKEN_KEYWORD_UNREACHABLE) \
+ TAG(TOKEN_KEYWORD_USINGNAMESPACE) \
+ TAG(TOKEN_KEYWORD_VAR) \
+ TAG(TOKEN_KEYWORD_VOLATILE) \
+ TAG(TOKEN_KEYWORD_WHILE)
#define TOKENIZER_GENERATE_ENUM(ENUM) ENUM,
#define TOKENIZER_GENERATE_CASE(ENUM) \
diff --git a/tokenizer_test.zig b/tokenizer_test.zig
@@ -10,128 +10,128 @@ const c = @cImport({
fn zigToken(token: c_uint) Token.Tag {
return switch (token) {
- c.TOKENIZER_TAG_INVALID => .invalid,
- c.TOKENIZER_TAG_INVALID_PERIODASTERISKS => .invalid_periodasterisks,
- c.TOKENIZER_TAG_IDENTIFIER => .identifier,
- c.TOKENIZER_TAG_STRING_LITERAL => .string_literal,
- c.TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE => .multiline_string_literal_line,
- c.TOKENIZER_TAG_CHAR_LITERAL => .char_literal,
- c.TOKENIZER_TAG_EOF => .eof,
- c.TOKENIZER_TAG_BUILTIN => .builtin,
- c.TOKENIZER_TAG_BANG => .bang,
- c.TOKENIZER_TAG_PIPE => .pipe,
- c.TOKENIZER_TAG_PIPE_PIPE => .pipe_pipe,
- c.TOKENIZER_TAG_PIPE_EQUAL => .pipe_equal,
- c.TOKENIZER_TAG_EQUAL => .equal,
- c.TOKENIZER_TAG_EQUAL_EQUAL => .equal_equal,
- c.TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT => .equal_angle_bracket_right,
- c.TOKENIZER_TAG_BANG_EQUAL => .bang_equal,
- c.TOKENIZER_TAG_L_PAREN => .l_paren,
- c.TOKENIZER_TAG_R_PAREN => .r_paren,
- c.TOKENIZER_TAG_SEMICOLON => .semicolon,
- c.TOKENIZER_TAG_PERCENT => .percent,
- c.TOKENIZER_TAG_PERCENT_EQUAL => .percent_equal,
- c.TOKENIZER_TAG_L_BRACE => .l_brace,
- c.TOKENIZER_TAG_R_BRACE => .r_brace,
- c.TOKENIZER_TAG_L_BRACKET => .l_bracket,
- c.TOKENIZER_TAG_R_BRACKET => .r_bracket,
- c.TOKENIZER_TAG_PERIOD => .period,
- c.TOKENIZER_TAG_PERIOD_ASTERISK => .period_asterisk,
- c.TOKENIZER_TAG_ELLIPSIS2 => .ellipsis2,
- c.TOKENIZER_TAG_ELLIPSIS3 => .ellipsis3,
- c.TOKENIZER_TAG_CARET => .caret,
- c.TOKENIZER_TAG_CARET_EQUAL => .caret_equal,
- c.TOKENIZER_TAG_PLUS => .plus,
- c.TOKENIZER_TAG_PLUS_PLUS => .plus_plus,
- c.TOKENIZER_TAG_PLUS_EQUAL => .plus_equal,
- c.TOKENIZER_TAG_PLUS_PERCENT => .plus_percent,
- c.TOKENIZER_TAG_PLUS_PERCENT_EQUAL => .plus_percent_equal,
- c.TOKENIZER_TAG_PLUS_PIPE => .plus_pipe,
- c.TOKENIZER_TAG_PLUS_PIPE_EQUAL => .plus_pipe_equal,
- c.TOKENIZER_TAG_MINUS => .minus,
- c.TOKENIZER_TAG_MINUS_EQUAL => .minus_equal,
- c.TOKENIZER_TAG_MINUS_PERCENT => .minus_percent,
- c.TOKENIZER_TAG_MINUS_PERCENT_EQUAL => .minus_percent_equal,
- c.TOKENIZER_TAG_MINUS_PIPE => .minus_pipe,
- c.TOKENIZER_TAG_MINUS_PIPE_EQUAL => .minus_pipe_equal,
- c.TOKENIZER_TAG_ASTERISK => .asterisk,
- c.TOKENIZER_TAG_ASTERISK_EQUAL => .asterisk_equal,
- c.TOKENIZER_TAG_ASTERISK_ASTERISK => .asterisk_asterisk,
- c.TOKENIZER_TAG_ASTERISK_PERCENT => .asterisk_percent,
- c.TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL => .asterisk_percent_equal,
- c.TOKENIZER_TAG_ASTERISK_PIPE => .asterisk_pipe,
- c.TOKENIZER_TAG_ASTERISK_PIPE_EQUAL => .asterisk_pipe_equal,
- c.TOKENIZER_TAG_ARROW => .arrow,
- c.TOKENIZER_TAG_COLON => .colon,
- c.TOKENIZER_TAG_SLASH => .slash,
- c.TOKENIZER_TAG_SLASH_EQUAL => .slash_equal,
- c.TOKENIZER_TAG_COMMA => .comma,
- c.TOKENIZER_TAG_AMPERSAND => .ampersand,
- c.TOKENIZER_TAG_AMPERSAND_EQUAL => .ampersand_equal,
- c.TOKENIZER_TAG_QUESTION_MARK => .question_mark,
- c.TOKENIZER_TAG_ANGLE_BRACKET_LEFT => .angle_bracket_left,
- c.TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_left_equal,
- c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT => .angle_bracket_angle_bracket_left,
- c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_angle_bracket_left_equal,
- c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE => .angle_bracket_angle_bracket_left_pipe,
- c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL => .angle_bracket_angle_bracket_left_pipe_equal,
- c.TOKENIZER_TAG_ANGLE_BRACKET_RIGHT => .angle_bracket_right,
- c.TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_right_equal,
- c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT => .angle_bracket_angle_bracket_right,
- c.TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_angle_bracket_right_equal,
- c.TOKENIZER_TAG_TILDE => .tilde,
- c.TOKENIZER_TAG_NUMBER_LITERAL => .number_literal,
- c.TOKENIZER_TAG_DOC_COMMENT => .doc_comment,
- c.TOKENIZER_TAG_CONTAINER_DOC_COMMENT => .container_doc_comment,
- c.TOKENIZER_TAG_KEYWORD_ADDRSPACE => .keyword_addrspace,
- c.TOKENIZER_TAG_KEYWORD_ALIGN => .keyword_align,
- c.TOKENIZER_TAG_KEYWORD_ALLOWZERO => .keyword_allowzero,
- c.TOKENIZER_TAG_KEYWORD_AND => .keyword_and,
- c.TOKENIZER_TAG_KEYWORD_ANYFRAME => .keyword_anyframe,
- c.TOKENIZER_TAG_KEYWORD_ANYTYPE => .keyword_anytype,
- c.TOKENIZER_TAG_KEYWORD_ASM => .keyword_asm,
- c.TOKENIZER_TAG_KEYWORD_ASYNC => .keyword_async,
- c.TOKENIZER_TAG_KEYWORD_AWAIT => .keyword_await,
- c.TOKENIZER_TAG_KEYWORD_BREAK => .keyword_break,
- c.TOKENIZER_TAG_KEYWORD_CALLCONV => .keyword_callconv,
- c.TOKENIZER_TAG_KEYWORD_CATCH => .keyword_catch,
- c.TOKENIZER_TAG_KEYWORD_COMPTIME => .keyword_comptime,
- c.TOKENIZER_TAG_KEYWORD_CONST => .keyword_const,
- c.TOKENIZER_TAG_KEYWORD_CONTINUE => .keyword_continue,
- c.TOKENIZER_TAG_KEYWORD_DEFER => .keyword_defer,
- c.TOKENIZER_TAG_KEYWORD_ELSE => .keyword_else,
- c.TOKENIZER_TAG_KEYWORD_ENUM => .keyword_enum,
- c.TOKENIZER_TAG_KEYWORD_ERRDEFER => .keyword_errdefer,
- c.TOKENIZER_TAG_KEYWORD_ERROR => .keyword_error,
- c.TOKENIZER_TAG_KEYWORD_EXPORT => .keyword_export,
- c.TOKENIZER_TAG_KEYWORD_EXTERN => .keyword_extern,
- c.TOKENIZER_TAG_KEYWORD_FN => .keyword_fn,
- c.TOKENIZER_TAG_KEYWORD_FOR => .keyword_for,
- c.TOKENIZER_TAG_KEYWORD_IF => .keyword_if,
- c.TOKENIZER_TAG_KEYWORD_INLINE => .keyword_inline,
- c.TOKENIZER_TAG_KEYWORD_NOALIAS => .keyword_noalias,
- c.TOKENIZER_TAG_KEYWORD_NOINLINE => .keyword_noinline,
- c.TOKENIZER_TAG_KEYWORD_NOSUSPEND => .keyword_nosuspend,
- c.TOKENIZER_TAG_KEYWORD_OPAQUE => .keyword_opaque,
- c.TOKENIZER_TAG_KEYWORD_OR => .keyword_or,
- c.TOKENIZER_TAG_KEYWORD_ORELSE => .keyword_orelse,
- c.TOKENIZER_TAG_KEYWORD_PACKED => .keyword_packed,
- c.TOKENIZER_TAG_KEYWORD_PUB => .keyword_pub,
- c.TOKENIZER_TAG_KEYWORD_RESUME => .keyword_resume,
- c.TOKENIZER_TAG_KEYWORD_RETURN => .keyword_return,
- c.TOKENIZER_TAG_KEYWORD_LINKSECTION => .keyword_linksection,
- c.TOKENIZER_TAG_KEYWORD_STRUCT => .keyword_struct,
- c.TOKENIZER_TAG_KEYWORD_SUSPEND => .keyword_suspend,
- c.TOKENIZER_TAG_KEYWORD_SWITCH => .keyword_switch,
- c.TOKENIZER_TAG_KEYWORD_TEST => .keyword_test,
- c.TOKENIZER_TAG_KEYWORD_THREADLOCAL => .keyword_threadlocal,
- c.TOKENIZER_TAG_KEYWORD_TRY => .keyword_try,
- c.TOKENIZER_TAG_KEYWORD_UNION => .keyword_union,
- c.TOKENIZER_TAG_KEYWORD_UNREACHABLE => .keyword_unreachable,
- c.TOKENIZER_TAG_KEYWORD_USINGNAMESPACE => .keyword_usingnamespace,
- c.TOKENIZER_TAG_KEYWORD_VAR => .keyword_var,
- c.TOKENIZER_TAG_KEYWORD_VOLATILE => .keyword_volatile,
- c.TOKENIZER_TAG_KEYWORD_WHILE => .keyword_while,
+ c.TOKEN_INVALID => .invalid,
+ c.TOKEN_INVALID_PERIODASTERISKS => .invalid_periodasterisks,
+ c.TOKEN_IDENTIFIER => .identifier,
+ c.TOKEN_STRING_LITERAL => .string_literal,
+ c.TOKEN_MULTILINE_STRING_LITERAL_LINE => .multiline_string_literal_line,
+ c.TOKEN_CHAR_LITERAL => .char_literal,
+ c.TOKEN_EOF => .eof,
+ c.TOKEN_BUILTIN => .builtin,
+ c.TOKEN_BANG => .bang,
+ c.TOKEN_PIPE => .pipe,
+ c.TOKEN_PIPE_PIPE => .pipe_pipe,
+ c.TOKEN_PIPE_EQUAL => .pipe_equal,
+ c.TOKEN_EQUAL => .equal,
+ c.TOKEN_EQUAL_EQUAL => .equal_equal,
+ c.TOKEN_EQUAL_ANGLE_BRACKET_RIGHT => .equal_angle_bracket_right,
+ c.TOKEN_BANG_EQUAL => .bang_equal,
+ c.TOKEN_L_PAREN => .l_paren,
+ c.TOKEN_R_PAREN => .r_paren,
+ c.TOKEN_SEMICOLON => .semicolon,
+ c.TOKEN_PERCENT => .percent,
+ c.TOKEN_PERCENT_EQUAL => .percent_equal,
+ c.TOKEN_L_BRACE => .l_brace,
+ c.TOKEN_R_BRACE => .r_brace,
+ c.TOKEN_L_BRACKET => .l_bracket,
+ c.TOKEN_R_BRACKET => .r_bracket,
+ c.TOKEN_PERIOD => .period,
+ c.TOKEN_PERIOD_ASTERISK => .period_asterisk,
+ c.TOKEN_ELLIPSIS2 => .ellipsis2,
+ c.TOKEN_ELLIPSIS3 => .ellipsis3,
+ c.TOKEN_CARET => .caret,
+ c.TOKEN_CARET_EQUAL => .caret_equal,
+ c.TOKEN_PLUS => .plus,
+ c.TOKEN_PLUS_PLUS => .plus_plus,
+ c.TOKEN_PLUS_EQUAL => .plus_equal,
+ c.TOKEN_PLUS_PERCENT => .plus_percent,
+ c.TOKEN_PLUS_PERCENT_EQUAL => .plus_percent_equal,
+ c.TOKEN_PLUS_PIPE => .plus_pipe,
+ c.TOKEN_PLUS_PIPE_EQUAL => .plus_pipe_equal,
+ c.TOKEN_MINUS => .minus,
+ c.TOKEN_MINUS_EQUAL => .minus_equal,
+ c.TOKEN_MINUS_PERCENT => .minus_percent,
+ c.TOKEN_MINUS_PERCENT_EQUAL => .minus_percent_equal,
+ c.TOKEN_MINUS_PIPE => .minus_pipe,
+ c.TOKEN_MINUS_PIPE_EQUAL => .minus_pipe_equal,
+ c.TOKEN_ASTERISK => .asterisk,
+ c.TOKEN_ASTERISK_EQUAL => .asterisk_equal,
+ c.TOKEN_ASTERISK_ASTERISK => .asterisk_asterisk,
+ c.TOKEN_ASTERISK_PERCENT => .asterisk_percent,
+ c.TOKEN_ASTERISK_PERCENT_EQUAL => .asterisk_percent_equal,
+ c.TOKEN_ASTERISK_PIPE => .asterisk_pipe,
+ c.TOKEN_ASTERISK_PIPE_EQUAL => .asterisk_pipe_equal,
+ c.TOKEN_ARROW => .arrow,
+ c.TOKEN_COLON => .colon,
+ c.TOKEN_SLASH => .slash,
+ c.TOKEN_SLASH_EQUAL => .slash_equal,
+ c.TOKEN_COMMA => .comma,
+ c.TOKEN_AMPERSAND => .ampersand,
+ c.TOKEN_AMPERSAND_EQUAL => .ampersand_equal,
+ c.TOKEN_QUESTION_MARK => .question_mark,
+ c.TOKEN_ANGLE_BRACKET_LEFT => .angle_bracket_left,
+ c.TOKEN_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_left_equal,
+ c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT => .angle_bracket_angle_bracket_left,
+ c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL => .angle_bracket_angle_bracket_left_equal,
+ c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE => .angle_bracket_angle_bracket_left_pipe,
+ c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL => .angle_bracket_angle_bracket_left_pipe_equal,
+ c.TOKEN_ANGLE_BRACKET_RIGHT => .angle_bracket_right,
+ c.TOKEN_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_right_equal,
+ c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT => .angle_bracket_angle_bracket_right,
+ c.TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL => .angle_bracket_angle_bracket_right_equal,
+ c.TOKEN_TILDE => .tilde,
+ c.TOKEN_NUMBER_LITERAL => .number_literal,
+ c.TOKEN_DOC_COMMENT => .doc_comment,
+ c.TOKEN_CONTAINER_DOC_COMMENT => .container_doc_comment,
+ c.TOKEN_KEYWORD_ADDRSPACE => .keyword_addrspace,
+ c.TOKEN_KEYWORD_ALIGN => .keyword_align,
+ c.TOKEN_KEYWORD_ALLOWZERO => .keyword_allowzero,
+ c.TOKEN_KEYWORD_AND => .keyword_and,
+ c.TOKEN_KEYWORD_ANYFRAME => .keyword_anyframe,
+ c.TOKEN_KEYWORD_ANYTYPE => .keyword_anytype,
+ c.TOKEN_KEYWORD_ASM => .keyword_asm,
+ c.TOKEN_KEYWORD_ASYNC => .keyword_async,
+ c.TOKEN_KEYWORD_AWAIT => .keyword_await,
+ c.TOKEN_KEYWORD_BREAK => .keyword_break,
+ c.TOKEN_KEYWORD_CALLCONV => .keyword_callconv,
+ c.TOKEN_KEYWORD_CATCH => .keyword_catch,
+ c.TOKEN_KEYWORD_COMPTIME => .keyword_comptime,
+ c.TOKEN_KEYWORD_CONST => .keyword_const,
+ c.TOKEN_KEYWORD_CONTINUE => .keyword_continue,
+ c.TOKEN_KEYWORD_DEFER => .keyword_defer,
+ c.TOKEN_KEYWORD_ELSE => .keyword_else,
+ c.TOKEN_KEYWORD_ENUM => .keyword_enum,
+ c.TOKEN_KEYWORD_ERRDEFER => .keyword_errdefer,
+ c.TOKEN_KEYWORD_ERROR => .keyword_error,
+ c.TOKEN_KEYWORD_EXPORT => .keyword_export,
+ c.TOKEN_KEYWORD_EXTERN => .keyword_extern,
+ c.TOKEN_KEYWORD_FN => .keyword_fn,
+ c.TOKEN_KEYWORD_FOR => .keyword_for,
+ c.TOKEN_KEYWORD_IF => .keyword_if,
+ c.TOKEN_KEYWORD_INLINE => .keyword_inline,
+ c.TOKEN_KEYWORD_NOALIAS => .keyword_noalias,
+ c.TOKEN_KEYWORD_NOINLINE => .keyword_noinline,
+ c.TOKEN_KEYWORD_NOSUSPEND => .keyword_nosuspend,
+ c.TOKEN_KEYWORD_OPAQUE => .keyword_opaque,
+ c.TOKEN_KEYWORD_OR => .keyword_or,
+ c.TOKEN_KEYWORD_ORELSE => .keyword_orelse,
+ c.TOKEN_KEYWORD_PACKED => .keyword_packed,
+ c.TOKEN_KEYWORD_PUB => .keyword_pub,
+ c.TOKEN_KEYWORD_RESUME => .keyword_resume,
+ c.TOKEN_KEYWORD_RETURN => .keyword_return,
+ c.TOKEN_KEYWORD_LINKSECTION => .keyword_linksection,
+ c.TOKEN_KEYWORD_STRUCT => .keyword_struct,
+ c.TOKEN_KEYWORD_SUSPEND => .keyword_suspend,
+ c.TOKEN_KEYWORD_SWITCH => .keyword_switch,
+ c.TOKEN_KEYWORD_TEST => .keyword_test,
+ c.TOKEN_KEYWORD_THREADLOCAL => .keyword_threadlocal,
+ c.TOKEN_KEYWORD_TRY => .keyword_try,
+ c.TOKEN_KEYWORD_UNION => .keyword_union,
+ c.TOKEN_KEYWORD_UNREACHABLE => .keyword_unreachable,
+ c.TOKEN_KEYWORD_USINGNAMESPACE => .keyword_usingnamespace,
+ c.TOKEN_KEYWORD_VAR => .keyword_var,
+ c.TOKEN_KEYWORD_VOLATILE => .keyword_volatile,
+ c.TOKEN_KEYWORD_WHILE => .keyword_while,
else => undefined,
};
}
