motiejus/zig0
1
Fork 0
Code Releases Activity

replace TOKENIZER_TAG_ with TOKEN_

Browse Source
This commit is contained in:
Motiejus Jakštys
2024-12-30 22:38:17 +02:00
parent d551ba3d12
commit 3264d1747e
5 changed files with 511 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files

288
tokenizer.c
View File

@@ -19,55 +19,55 @@ const char* tokenizerGetTagString(TokenizerTag tag) {
}
const KeywordMap keywords[] = {
{ "addrspace", TOKENIZER_TAG_KEYWORD_ADDRSPACE },
{ "align", TOKENIZER_TAG_KEYWORD_ALIGN },
{ "allowzero", TOKENIZER_TAG_KEYWORD_ALLOWZERO },
{ "and", TOKENIZER_TAG_KEYWORD_AND },
{ "anyframe", TOKENIZER_TAG_KEYWORD_ANYFRAME },
{ "anytype", TOKENIZER_TAG_KEYWORD_ANYTYPE },
{ "asm", TOKENIZER_TAG_KEYWORD_ASM },
{ "async", TOKENIZER_TAG_KEYWORD_ASYNC },
{ "await", TOKENIZER_TAG_KEYWORD_AWAIT },
{ "break", TOKENIZER_TAG_KEYWORD_BREAK },
{ "callconv", TOKENIZER_TAG_KEYWORD_CALLCONV },
{ "catch", TOKENIZER_TAG_KEYWORD_CATCH },
{ "comptime", TOKENIZER_TAG_KEYWORD_COMPTIME },
{ "const", TOKENIZER_TAG_KEYWORD_CONST },
{ "continue", TOKENIZER_TAG_KEYWORD_CONTINUE },
{ "defer", TOKENIZER_TAG_KEYWORD_DEFER },
{ "else", TOKENIZER_TAG_KEYWORD_ELSE },
{ "enum", TOKENIZER_TAG_KEYWORD_ENUM },
{ "errdefer", TOKENIZER_TAG_KEYWORD_ERRDEFER },
{ "error", TOKENIZER_TAG_KEYWORD_ERROR },
{ "export", TOKENIZER_TAG_KEYWORD_EXPORT },
{ "extern", TOKENIZER_TAG_KEYWORD_EXTERN },
{ "fn", TOKENIZER_TAG_KEYWORD_FN },
{ "for", TOKENIZER_TAG_KEYWORD_FOR },
{ "if", TOKENIZER_TAG_KEYWORD_IF },
{ "inline", TOKENIZER_TAG_KEYWORD_INLINE },
{ "linksection", TOKENIZER_TAG_KEYWORD_LINKSECTION },
{ "noalias", TOKENIZER_TAG_KEYWORD_NOALIAS },
{ "noinline", TOKENIZER_TAG_KEYWORD_NOINLINE },
{ "nosuspend", TOKENIZER_TAG_KEYWORD_NOSUSPEND },
{ "opaque", TOKENIZER_TAG_KEYWORD_OPAQUE },
{ "or", TOKENIZER_TAG_KEYWORD_OR },
{ "orelse", TOKENIZER_TAG_KEYWORD_ORELSE },
{ "packed", TOKENIZER_TAG_KEYWORD_PACKED },
{ "pub", TOKENIZER_TAG_KEYWORD_PUB },
{ "resume", TOKENIZER_TAG_KEYWORD_RESUME },
{ "return", TOKENIZER_TAG_KEYWORD_RETURN },
{ "struct", TOKENIZER_TAG_KEYWORD_STRUCT },
{ "suspend", TOKENIZER_TAG_KEYWORD_SUSPEND },
{ "switch", TOKENIZER_TAG_KEYWORD_SWITCH },
{ "test", TOKENIZER_TAG_KEYWORD_TEST },
{ "threadlocal", TOKENIZER_TAG_KEYWORD_THREADLOCAL },
{ "try", TOKENIZER_TAG_KEYWORD_TRY },
{ "union", TOKENIZER_TAG_KEYWORD_UNION },
{ "unreachable", TOKENIZER_TAG_KEYWORD_UNREACHABLE },
{ "usingnamespace", TOKENIZER_TAG_KEYWORD_USINGNAMESPACE },
{ "var", TOKENIZER_TAG_KEYWORD_VAR },
{ "volatile", TOKENIZER_TAG_KEYWORD_VOLATILE },
{ "while", TOKENIZER_TAG_KEYWORD_WHILE }
{ "addrspace", TOKEN_KEYWORD_ADDRSPACE },
{ "align", TOKEN_KEYWORD_ALIGN },
{ "allowzero", TOKEN_KEYWORD_ALLOWZERO },
{ "and", TOKEN_KEYWORD_AND },
{ "anyframe", TOKEN_KEYWORD_ANYFRAME },
{ "anytype", TOKEN_KEYWORD_ANYTYPE },
{ "asm", TOKEN_KEYWORD_ASM },
{ "async", TOKEN_KEYWORD_ASYNC },
{ "await", TOKEN_KEYWORD_AWAIT },
{ "break", TOKEN_KEYWORD_BREAK },
{ "callconv", TOKEN_KEYWORD_CALLCONV },
{ "catch", TOKEN_KEYWORD_CATCH },
{ "comptime", TOKEN_KEYWORD_COMPTIME },
{ "const", TOKEN_KEYWORD_CONST },
{ "continue", TOKEN_KEYWORD_CONTINUE },
{ "defer", TOKEN_KEYWORD_DEFER },
{ "else", TOKEN_KEYWORD_ELSE },
{ "enum", TOKEN_KEYWORD_ENUM },
{ "errdefer", TOKEN_KEYWORD_ERRDEFER },
{ "error", TOKEN_KEYWORD_ERROR },
{ "export", TOKEN_KEYWORD_EXPORT },
{ "extern", TOKEN_KEYWORD_EXTERN },
{ "fn", TOKEN_KEYWORD_FN },
{ "for", TOKEN_KEYWORD_FOR },
{ "if", TOKEN_KEYWORD_IF },
{ "inline", TOKEN_KEYWORD_INLINE },
{ "linksection", TOKEN_KEYWORD_LINKSECTION },
{ "noalias", TOKEN_KEYWORD_NOALIAS },
{ "noinline", TOKEN_KEYWORD_NOINLINE },
{ "nosuspend", TOKEN_KEYWORD_NOSUSPEND },
{ "opaque", TOKEN_KEYWORD_OPAQUE },
{ "or", TOKEN_KEYWORD_OR },
{ "orelse", TOKEN_KEYWORD_ORELSE },
{ "packed", TOKEN_KEYWORD_PACKED },
{ "pub", TOKEN_KEYWORD_PUB },
{ "resume", TOKEN_KEYWORD_RESUME },
{ "return", TOKEN_KEYWORD_RETURN },
{ "struct", TOKEN_KEYWORD_STRUCT },
{ "suspend", TOKEN_KEYWORD_SUSPEND },
{ "switch", TOKEN_KEYWORD_SWITCH },
{ "test", TOKEN_KEYWORD_TEST },
{ "threadlocal", TOKEN_KEYWORD_THREADLOCAL },
{ "try", TOKEN_KEYWORD_TRY },
{ "union", TOKEN_KEYWORD_UNION },
{ "unreachable", TOKEN_KEYWORD_UNREACHABLE },
{ "usingnamespace", TOKEN_KEYWORD_USINGNAMESPACE },
{ "var", TOKEN_KEYWORD_VAR },
{ "volatile", TOKEN_KEYWORD_VOLATILE },
{ "while", TOKEN_KEYWORD_WHILE }
};
// TODO binary search
@@ -80,13 +80,13 @@ static TokenizerTag getKeyword(const char* bytes, const uint32_t len) {
if (len == klen) {
return keywords[i].tag;
} else {
return TOKENIZER_TAG_INVALID;
return TOKEN_INVALID;
}
} else if (cmp < 0) {
return TOKENIZER_TAG_INVALID;
return TOKEN_INVALID;
}
}
return TOKENIZER_TAG_INVALID;
return TOKEN_INVALID;
}
Tokenizer tokenizerInit(const char* buffer, const uint32_t len) {
@@ -99,7 +99,7 @@ Tokenizer tokenizerInit(const char* buffer, const uint32_t len) {
TokenizerToken tokenizerNext(Tokenizer* self) {
TokenizerToken result = (TokenizerToken) {
.tag = TOKENIZER_TAG_INVALID,
.tag = TOKEN_INVALID,
.loc = {
.start = 0,
},
@@ -114,7 +114,7 @@ state:
case 0:
if (self->index == self->buffer_len) {
return (TokenizerToken) {
.tag = TOKENIZER_TAG_EOF,
.tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index,
@@ -132,17 +132,17 @@ state:
result.loc.start = self->index;
goto state;
case '"':
result.tag = TOKENIZER_TAG_STRING_LITERAL;
result.tag = TOKEN_STRING_LITERAL;
state = TOKENIZER_STATE_STRING_LITERAL;
goto state;
case '\'':
result.tag = TOKENIZER_TAG_CHAR_LITERAL;
result.tag = TOKEN_CHAR_LITERAL;
state = TOKENIZER_STATE_CHAR_LITERAL;
goto state;
case 'a' ... 'z':
case 'A' ... 'Z':
case '_':
result.tag = TOKENIZER_TAG_IDENTIFIER;
result.tag = TOKEN_IDENTIFIER;
state = TOKENIZER_STATE_IDENTIFIER;
goto state;
case '@':
@@ -158,35 +158,35 @@ state:
state = TOKENIZER_STATE_PIPE;
goto state;
case '(':
result.tag = TOKENIZER_TAG_L_PAREN;
result.tag = TOKEN_L_PAREN;
self->index++;
break;
case ')':
result.tag = TOKENIZER_TAG_R_PAREN;
result.tag = TOKEN_R_PAREN;
self->index++;
break;
case '[':
result.tag = TOKENIZER_TAG_L_BRACKET;
result.tag = TOKEN_L_BRACKET;
self->index++;
break;
case ']':
result.tag = TOKENIZER_TAG_R_BRACKET;
result.tag = TOKEN_R_BRACKET;
self->index++;
break;
case ';':
result.tag = TOKENIZER_TAG_SEMICOLON;
result.tag = TOKEN_SEMICOLON;
self->index++;
break;
case ',':
result.tag = TOKENIZER_TAG_COMMA;
result.tag = TOKEN_COMMA;
self->index++;
break;
case '?':
result.tag = TOKENIZER_TAG_QUESTION_MARK;
result.tag = TOKEN_QUESTION_MARK;
self->index++;
break;
case ':':
result.tag = TOKENIZER_TAG_COLON;
result.tag = TOKEN_COLON;
self->index++;
break;
case '%':
@@ -208,19 +208,19 @@ state:
state = TOKENIZER_STATE_CARET;
goto state;
case '\\':
result.tag = TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE;
result.tag = TOKEN_MULTILINE_STRING_LITERAL_LINE;
state = TOKENIZER_STATE_BACKSLASH;
goto state;
case '{':
result.tag = TOKENIZER_TAG_L_BRACE;
result.tag = TOKEN_L_BRACE;
self->index++;
break;
case '}':
result.tag = TOKENIZER_TAG_R_BRACE;
result.tag = TOKEN_R_BRACE;
self->index++;
break;
case '~':
result.tag = TOKENIZER_TAG_TILDE;
result.tag = TOKEN_TILDE;
self->index++;
break;
case '.':
@@ -236,7 +236,7 @@ state:
state = TOKENIZER_STATE_AMPERSAND;
goto state;
case '0' ... '9':
result.tag = TOKENIZER_TAG_NUMBER_LITERAL;
result.tag = TOKEN_NUMBER_LITERAL;
self->index++;
state = TOKENIZER_STATE_INT;
goto state;
@@ -251,7 +251,7 @@ state:
switch (self->buffer[self->index]) {
case 0:
if (self->index == self->buffer_len) {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
@@ -273,14 +273,14 @@ state:
switch (self->buffer[self->index]) {
case 0:
if (self->index == self->buffer_len) {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_INVALID;
@@ -293,16 +293,16 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '"':
result.tag = TOKENIZER_TAG_IDENTIFIER;
result.tag = TOKEN_IDENTIFIER;
state = TOKENIZER_STATE_STRING_LITERAL;
goto state;
case 'a' ... 'z':
case 'A' ... 'Z':
case '_':
result.tag = TOKENIZER_TAG_BUILTIN;
result.tag = TOKEN_BUILTIN;
state = TOKENIZER_STATE_BUILTIN;
goto state;
default:
@@ -315,11 +315,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_AMPERSAND_EQUAL;
result.tag = TOKEN_AMPERSAND_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_AMPERSAND;
result.tag = TOKEN_AMPERSAND;
break;
}
break;
@@ -328,11 +328,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ASTERISK_EQUAL;
result.tag = TOKEN_ASTERISK_EQUAL;
self->index++;
break;
case '*':
result.tag = TOKENIZER_TAG_ASTERISK_ASTERISK;
result.tag = TOKEN_ASTERISK_ASTERISK;
self->index++;
break;
case '%':
@@ -342,7 +342,7 @@ state:
state = TOKENIZER_STATE_ASTERISK_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_ASTERISK;
result.tag = TOKEN_ASTERISK;
break;
}
break;
@@ -351,11 +351,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL;
result.tag = TOKEN_ASTERISK_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ASTERISK_PERCENT;
result.tag = TOKEN_ASTERISK_PERCENT;
break;
}
break;
@@ -364,11 +364,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ASTERISK_PIPE_EQUAL;
result.tag = TOKEN_ASTERISK_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ASTERISK_PIPE;
result.tag = TOKEN_ASTERISK_PIPE;
break;
}
break;
@@ -377,11 +377,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PERCENT_EQUAL;
result.tag = TOKEN_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PERCENT;
result.tag = TOKEN_PERCENT;
break;
}
break;
@@ -390,11 +390,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PLUS_EQUAL;
result.tag = TOKEN_PLUS_EQUAL;
self->index++;
break;
case '+':
result.tag = TOKENIZER_TAG_PLUS_PLUS;
result.tag = TOKEN_PLUS_PLUS;
self->index++;
break;
case '%':
@@ -404,7 +404,7 @@ state:
state = TOKENIZER_STATE_PLUS_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_PLUS;
result.tag = TOKEN_PLUS;
break;
}
break;
@@ -413,11 +413,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PLUS_PERCENT_EQUAL;
result.tag = TOKEN_PLUS_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PLUS_PERCENT;
result.tag = TOKEN_PLUS_PERCENT;
break;
}
break;
@@ -426,11 +426,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PLUS_PIPE_EQUAL;
result.tag = TOKEN_PLUS_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PLUS_PIPE;
result.tag = TOKEN_PLUS_PIPE;
break;
}
break;
@@ -439,11 +439,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_CARET_EQUAL;
result.tag = TOKEN_CARET_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_CARET;
result.tag = TOKEN_CARET;
break;
}
break;
@@ -461,7 +461,7 @@ state:
const char* start = self->buffer + result.loc.start;
uint32_t len = self->index - result.loc.start;
TokenizerTag tag = getKeyword(start, len);
if (tag != TOKENIZER_TAG_INVALID) {
if (tag != TOKEN_INVALID) {
result.tag = tag;
}
}
@@ -484,13 +484,13 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case 0:
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_MULTILINE_STRING_LITERAL_LINE;
goto state;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_INVALID;
@@ -506,11 +506,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_STRING_LITERAL_BACKSLASH;
@@ -534,7 +534,7 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
default:
state = TOKENIZER_STATE_STRING_LITERAL;
@@ -550,11 +550,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case '\\':
state = TOKENIZER_STATE_CHAR_LITERAL_BACKSLASH;
@@ -581,11 +581,11 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
} else {
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
}
break;
case '\n':
result.tag = TOKENIZER_TAG_INVALID;
result.tag = TOKEN_INVALID;
break;
case 0x01 ... 0x09:
case 0x0b ... 0x1f:
@@ -631,11 +631,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_BANG_EQUAL;
result.tag = TOKEN_BANG_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_BANG;
result.tag = TOKEN_BANG;
break;
}
break;
@@ -644,15 +644,15 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_PIPE_EQUAL;
result.tag = TOKEN_PIPE_EQUAL;
self->index++;
break;
case '|':
result.tag = TOKENIZER_TAG_PIPE_PIPE;
result.tag = TOKEN_PIPE_PIPE;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_PIPE;
result.tag = TOKEN_PIPE;
break;
}
break;
@@ -661,15 +661,15 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_EQUAL_EQUAL;
result.tag = TOKEN_EQUAL_EQUAL;
self->index++;
break;
case '>':
result.tag = TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT;
result.tag = TOKEN_EQUAL_ANGLE_BRACKET_RIGHT;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_EQUAL;
result.tag = TOKEN_EQUAL;
break;
}
break;
@@ -678,11 +678,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '>':
result.tag = TOKENIZER_TAG_ARROW;
result.tag = TOKEN_ARROW;
self->index++;
break;
case '=':
result.tag = TOKENIZER_TAG_MINUS_EQUAL;
result.tag = TOKEN_MINUS_EQUAL;
self->index++;
break;
case '%':
@@ -692,7 +692,7 @@ state:
state = TOKENIZER_STATE_MINUS_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_MINUS;
result.tag = TOKEN_MINUS;
break;
}
break;
@@ -701,11 +701,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_MINUS_PERCENT_EQUAL;
result.tag = TOKEN_MINUS_PERCENT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_MINUS_PERCENT;
result.tag = TOKEN_MINUS_PERCENT;
break;
}
break;
@@ -714,11 +714,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_MINUS_PIPE_EQUAL;
result.tag = TOKEN_MINUS_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_MINUS_PIPE;
result.tag = TOKEN_MINUS_PIPE;
break;
}
break;
@@ -730,11 +730,11 @@ state:
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
goto state;
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_LEFT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_LEFT;
result.tag = TOKEN_ANGLE_BRACKET_LEFT;
break;
}
break;
@@ -743,14 +743,14 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL;
self->index++;
break;
case '|':
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
goto state;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT;
break;
}
break;
@@ -759,11 +759,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE;
break;
}
break;
@@ -775,11 +775,11 @@ state:
state = TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
goto state;
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_RIGHT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_RIGHT;
result.tag = TOKEN_ANGLE_BRACKET_RIGHT;
break;
}
break;
@@ -788,11 +788,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '=':
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
result.tag = TOKEN_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT;
break;
}
break;
@@ -807,7 +807,7 @@ state:
state = TOKENIZER_STATE_PERIOD_ASTERISK;
goto state;
default:
result.tag = TOKENIZER_TAG_PERIOD;
result.tag = TOKEN_PERIOD;
break;
}
break;
@@ -816,11 +816,11 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '.':
result.tag = TOKENIZER_TAG_ELLIPSIS3;
result.tag = TOKEN_ELLIPSIS3;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_ELLIPSIS2;
result.tag = TOKEN_ELLIPSIS2;
break;
}
break;
@@ -829,10 +829,10 @@ state:
self->index++;
switch (self->buffer[self->index]) {
case '*':
result.tag = TOKENIZER_TAG_INVALID_PERIODASTERISKS;
result.tag = TOKEN_INVALID_PERIODASTERISKS;
break;
default:
result.tag = TOKENIZER_TAG_PERIOD_ASTERISK;
result.tag = TOKEN_PERIOD_ASTERISK;
break;
}
break;
@@ -844,11 +844,11 @@ state:
state = TOKENIZER_STATE_LINE_COMMENT_START;
goto state;
case '=':
result.tag = TOKENIZER_TAG_SLASH_EQUAL;
result.tag = TOKEN_SLASH_EQUAL;
self->index++;
break;
default:
result.tag = TOKENIZER_TAG_SLASH;
result.tag = TOKEN_SLASH;
break;
}
break;
@@ -862,7 +862,7 @@ state:
goto state;
} else {
return (TokenizerToken) {
.tag = TOKENIZER_TAG_EOF,
.tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index }
@@ -870,7 +870,7 @@ state:
}
break;
case '!':
result.tag = TOKENIZER_TAG_CONTAINER_DOC_COMMENT;
result.tag = TOKEN_CONTAINER_DOC_COMMENT;
state = TOKENIZER_STATE_DOC_COMMENT;
goto state;
case '\n':
@@ -901,11 +901,11 @@ state:
switch (self->buffer[self->index]) {
case 0:
case '\n':
result.tag = TOKENIZER_TAG_DOC_COMMENT;
result.tag = TOKEN_DOC_COMMENT;
break;
case '\r':
if (self->buffer[self->index + 1] == '\n') {
result.tag = TOKENIZER_TAG_DOC_COMMENT;
result.tag = TOKEN_DOC_COMMENT;
} else {
state = TOKENIZER_STATE_INVALID;
goto state;
@@ -921,7 +921,7 @@ state:
state = TOKENIZER_STATE_INVALID;
goto state;
default:
result.tag = TOKENIZER_TAG_DOC_COMMENT;
result.tag = TOKEN_DOC_COMMENT;
state = TOKENIZER_STATE_DOC_COMMENT;
goto state;
}
@@ -936,7 +936,7 @@ state:
goto state;
} else {
return (TokenizerToken) {
.tag = TOKENIZER_TAG_EOF,
.tag = TOKEN_EOF,
.loc = {
.start = self->index,
.end = self->index }