motiejus/zig
1
Fork 0
Code Releases Activity

stage1: tokenizer: backport recent stage2 changes

Browse Source 
This backports ec10595b65 stage2 tokenizer
logic to stage1.
This commit is contained in:
Andrew Kelley
2021-05-28 13:26:16 -07:00
parent 01e0a4b2f0
commit 9de2c1b76e
1 changed files with 2 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/stage1/tokenizer.cpp
View File

@@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) {
case DIGIT:
t.state = TokenizeState_float_fraction_dec;
break;
case ALPHA_EXCEPT_E:
default:
invalid_char_error(&t, c);
break;
default:
t.state = TokenizeState_start;
continue;
}
break;
case TokenizeState_num_dot_hex:
@@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) {
t.out->ids.last() = TokenIdFloatLiteral;
t.state = TokenizeState_float_fraction_hex;
break;
case ALPHA_EXCEPT_HEX_AND_P:
default:
invalid_char_error(&t, c);
break;
default:
t.state = TokenizeState_start;
continue;
}
break;
case TokenizeState_float_fraction_dec_no_underscore: