motiejus/zig
1
Fork 0
Code Releases Activity

introduce [*] for unknown length pointers

Browse Source 
See #770

Currently it does not have any different behavior than `*`
but it is now recommended to use `[*]` for unknown length
pointers to be future-proof.

Instead of [ * ] being separate tokens as the proposal
suggested, this commit implements `[*]` as a single token.
This commit is contained in:
Andrew Kelley
2018-06-02 04:03:25 -04:00
parent 7b386ea242
commit f06bce5dda
9 changed files with 87 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/tokenizer.cpp
View File

@@ -219,6 +219,8 @@ enum TokenizeState {
TokenizeStateSawAtSign,
TokenizeStateCharCode,
TokenizeStateError,
TokenizeStateLBracket,
TokenizeStateLBracketStar,
};
@@ -539,8 +541,8 @@ void tokenize(Buf *buf, Tokenization *out) {
end_token(&t);
break;
case '[':
t.state = TokenizeStateLBracket;
begin_token(&t, TokenIdLBracket);
end_token(&t);
break;
case ']':
begin_token(&t, TokenIdRBracket);
@@ -852,6 +854,30 @@ void tokenize(Buf *buf, Tokenization *out) {
continue;
}
break;
case TokenizeStateLBracket:
switch (c) {
case '*':
t.state = TokenizeStateLBracketStar;
set_token_id(&t, t.cur_tok, TokenIdBracketStarBracket);
break;
default:
// reinterpret as just an lbracket
t.pos -= 1;
end_token(&t);
t.state = TokenizeStateStart;
continue;
}
break;
case TokenizeStateLBracketStar:
switch (c) {
case ']':
end_token(&t);
t.state = TokenizeStateStart;
break;
default:
invalid_char_error(&t, c);
}
break;
case TokenizeStateSawPlusPercent:
switch (c) {
case '=':
@@ -1467,12 +1493,14 @@ void tokenize(Buf *buf, Tokenization *out) {
case TokenizeStateLineString:
case TokenizeStateLineStringEnd:
case TokenizeStateSawBarBar:
case TokenizeStateLBracket:
end_token(&t);
break;
case TokenizeStateSawDotDot:
case TokenizeStateSawBackslash:
case TokenizeStateLineStringContinue:
case TokenizeStateLineStringContinueC:
case TokenizeStateLBracketStar:
tokenize_error(&t, "unexpected EOF");
break;
case TokenizeStateLineComment:
@@ -1509,6 +1537,7 @@ const char * token_name(TokenId id) {
case TokenIdBitShiftRight: return ">>";
case TokenIdBitShiftRightEq: return ">>=";
case TokenIdBitXorEq: return "^=";
case TokenIdBracketStarBracket: return "[*]";
case TokenIdCharLiteral: return "CharLiteral";
case TokenIdCmpEq: return "==";
case TokenIdCmpGreaterOrEq: return ">=";