zig

commit 9de2c1b76e2b38e35bb17cd0a3943a61c730da76 (tree)
parent 01e0a4b2f06d60a58b3d2df410776b164d05c1ec
Author: Andrew Kelley <andrew@ziglang.org>
Date:   Fri, 28 May 2021 13:26:16 -0700

stage1: tokenizer: backport recent stage2 changes

This backports ec10595b65490126e1b7d231f655be6d7d198acf stage2 tokenizer
logic to stage1.

Diffstat:
M
src/stage1/tokenizer.cpp
 | 
10
++--------

1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp
@@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) {
                     case DIGIT:
                         t.state = TokenizeState_float_fraction_dec;
                         break;
-                    case ALPHA_EXCEPT_E:
+                    default:
                         invalid_char_error(&t, c);
                         break;
-                    default:
-                        t.state = TokenizeState_start;
-                        continue;
                 }
                 break;
             case TokenizeState_num_dot_hex:
@@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) {
                         t.out->ids.last() = TokenIdFloatLiteral;
                         t.state = TokenizeState_float_fraction_hex;
                         break;
-                    case ALPHA_EXCEPT_HEX_AND_P:
+                    default:
                         invalid_char_error(&t, c);
                         break;
-                    default:
-                        t.state = TokenizeState_start;
-                        continue;
                 }
                 break;
             case TokenizeState_float_fraction_dec_no_underscore: