motiejus/zig
1
Fork 0
Code Releases Activity

update the stage1 implementation to the new proposal

Browse Source 
See #3731
This commit is contained in:
Andrew Kelley
2019-11-23 04:45:35 -05:00
parent 6b623b5ea2
commit 7597735bad
21 changed files with 613 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/tokenizer.cpp
View File

@@ -222,10 +222,6 @@ enum TokenizeState {
TokenizeStateSawAtSign,
TokenizeStateCharCode,
TokenizeStateError,
TokenizeStateLBracket,
TokenizeStateLBracketStar,
TokenizeStateLBracketStarC,
TokenizeStateLBracketUnderscore,
};
@@ -480,8 +476,8 @@ void tokenize(Buf *buf, Tokenization *out) {
end_token(&t);
break;
case '[':
t.state = TokenizeStateLBracket;
begin_token(&t, TokenIdLBracket);
end_token(&t);
break;
case ']':
begin_token(&t, TokenIdRBracket);
@@ -775,62 +771,6 @@ void tokenize(Buf *buf, Tokenization *out) {
continue;
}
break;
case TokenizeStateLBracket:
switch (c) {
case '*':
t.state = TokenizeStateLBracketStar;
break;
case '_':
t.state = TokenizeStateLBracketUnderscore;
break;
default:
// reinterpret as just an lbracket
t.pos -= 1;
end_token(&t);
t.state = TokenizeStateStart;
continue;
}
break;
case TokenizeStateLBracketUnderscore:
switch (c) {
case ']':
set_token_id(&t, t.cur_tok, TokenIdBracketUnderscoreBracket);
end_token(&t);
t.state = TokenizeStateStart;
break;
default:
// reinterpret as just an lbracket
t.pos -= 2;
end_token(&t);
t.state = TokenizeStateStart;
continue;
}
break;
case TokenizeStateLBracketStar:
switch (c) {
case 'c':
t.state = TokenizeStateLBracketStarC;
set_token_id(&t, t.cur_tok, TokenIdBracketStarCBracket);
break;
case ']':
set_token_id(&t, t.cur_tok, TokenIdBracketStarBracket);
end_token(&t);
t.state = TokenizeStateStart;
break;
default:
invalid_char_error(&t, c);
}
break;
case TokenizeStateLBracketStarC:
switch (c) {
case ']':
end_token(&t);
t.state = TokenizeStateStart;
break;
default:
invalid_char_error(&t, c);
}
break;
case TokenizeStateSawPlusPercent:
switch (c) {
case '=':
@@ -1525,7 +1465,6 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateLineString:
case TokenizeStateLineStringEnd:
case TokenizeStateSawBarBar:
case TokenizeStateLBracket:
case TokenizeStateDocComment:
case TokenizeStateContainerDocComment:
end_token(&t);
@@ -1534,9 +1473,6 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateSawBackslash:
case TokenizeStateLineStringContinue:
case TokenizeStateLineStringContinueC:
case TokenizeStateLBracketStar:
case TokenizeStateLBracketStarC:
case TokenizeStateLBracketUnderscore:
tokenize_error(&t, "unexpected EOF");
break;
case TokenizeStateLineComment:
@@ -1576,8 +1512,6 @@ const char * token_name(TokenId id) {
case TokenIdBitShiftRight: return ">>";
case TokenIdBitShiftRightEq: return ">>=";
case TokenIdBitXorEq: return "^=";
case TokenIdBracketStarBracket: return "[*]";
case TokenIdBracketStarCBracket: return "[*c]";
case TokenIdCharLiteral: return "CharLiteral";
case TokenIdCmpEq: return "==";
case TokenIdCmpGreaterOrEq: return ">=";
@@ -1681,7 +1615,6 @@ const char * token_name(TokenId id) {
case TokenIdTimesPercent: return "*%";
case TokenIdTimesPercentEq: return "*%=";
case TokenIdBarBarEq: return "||=";
case TokenIdBracketUnderscoreBracket: return "[_]";
case TokenIdCount:
zig_unreachable();
}