motiejus/zig
1
Fork 0
Code Releases Activity

stage1 parser supports doc comments

Browse Source
This commit is contained in:
Andrew Kelley
2019-10-06 16:39:27 -04:00
parent 86171afb9b
commit 8e2c441b2e
8 changed files with 133 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/tokenizer.cpp
View File

@@ -196,6 +196,8 @@ enum TokenizeState {
TokenizeStateSawStar,
TokenizeStateSawStarPercent,
TokenizeStateSawSlash,
TokenizeStateSawSlash2,
TokenizeStateSawSlash3,
TokenizeStateSawBackslash,
TokenizeStateSawPercent,
TokenizeStateSawPlus,
@@ -206,6 +208,7 @@ enum TokenizeState {
TokenizeStateSawCaret,
TokenizeStateSawBar,
TokenizeStateSawBarBar,
TokenizeStateDocComment,
TokenizeStateLineComment,
TokenizeStateLineString,
TokenizeStateLineStringEnd,
@@ -910,8 +913,7 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateSawSlash:
switch (c) {
case '/':
cancel_token(&t);
t.state = TokenizeStateLineComment;
t.state = TokenizeStateSawSlash2;
break;
case '=':
set_token_id(&t, t.cur_tok, TokenIdDivEq);
@@ -925,6 +927,38 @@ void tokenize(Buf *buf, Tokenization *out) {
continue;
}
break;
case TokenizeStateSawSlash2:
switch (c) {
case '/':
t.state = TokenizeStateSawSlash3;
break;
case '\n':
cancel_token(&t);
t.state = TokenizeStateStart;
break;
default:
cancel_token(&t);
t.state = TokenizeStateLineComment;
break;
}
break;
case TokenizeStateSawSlash3:
switch (c) {
case '/':
cancel_token(&t);
t.state = TokenizeStateLineComment;
break;
case '\n':
set_token_id(&t, t.cur_tok, TokenIdDocComment);
end_token(&t);
t.state = TokenizeStateStart;
break;
default:
set_token_id(&t, t.cur_tok, TokenIdDocComment);
t.state = TokenizeStateDocComment;
break;
}
break;
case TokenizeStateSawBackslash:
switch (c) {
case '\\':
@@ -1004,6 +1038,17 @@ void tokenize(Buf *buf, Tokenization *out) {
break;
}
break;
case TokenizeStateDocComment:
switch (c) {
case '\n':
end_token(&t);
t.state = TokenizeStateStart;
break;
default:
// do nothing
break;
}
break;
case TokenizeStateSymbolFirstC:
switch (c) {
case '"':
@@ -1466,6 +1511,7 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateLineStringEnd:
case TokenizeStateSawBarBar:
case TokenizeStateLBracket:
case TokenizeStateDocComment:
end_token(&t);
break;
case TokenizeStateSawDotDot:
@@ -1478,6 +1524,8 @@ void tokenize(Buf *buf, Tokenization *out) {
tokenize_error(&t, "unexpected EOF");
break;
case TokenizeStateLineComment:
case TokenizeStateSawSlash2:
case TokenizeStateSawSlash3:
break;
}
if (t.state != TokenizeStateError) {
@@ -1524,6 +1572,7 @@ const char * token_name(TokenId id) {
case TokenIdComma: return ",";
case TokenIdDash: return "-";
case TokenIdDivEq: return "/=";
case TokenIdDocComment: return "DocComment";
case TokenIdDot: return ".";
case TokenIdEllipsis2: return "..";
case TokenIdEllipsis3: return "...";