sat-arithmetic: correctly tokenize <<|, <<|=

- set state rather than result.tag in tokenizer.zig
- add test to tokenizer.zig for <<, <<|, <<|=
This commit is contained in:
Travis Staloch
2021-09-08 15:58:37 -07:00
committed by Andrew Kelley
parent b9a95f2dd9
commit dcbc52ec85
3 changed files with 10 additions and 5 deletions

View File

@@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) {
t.state = TokenizeState_start;
break;
case '|':
// t.out->ids.last() = TokenIdBitShiftLeftPipe;
t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;
break;
default: