motiejus/zig

commit dcbc52ec85d00fbd3603c314aaaab98fb3866892 (tree)
parent b9a95f2dd94e6175322d3388c3936eb600ec90ea
Author: Travis Staloch <twostepted@gmail.com>
Date:   Wed,  8 Sep 2021 15:58:37 -0700

sat-arithmetic: correctly tokenize <<|, <<|=

- set state rather than result.tag in tokenizer.zig
- add test to tokenizer.zig for <<, <<|, <<|=

Diffstat:
M
lib/std/zig/tokenizer.zig
 | 
8
+++++++-
M
src/Air.zig
 | 
6
+++---
M
src/stage1/tokenizer.cpp
 | 
1
-

3 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
@@ -1007,7 +1007,7 @@ pub const Tokenizer = struct {
                         break;
                     },
                     '|' => {
-                        result.tag = .angle_bracket_angle_bracket_left_pipe;
+                        state = .angle_bracket_angle_bracket_left_pipe;
                     },
                     else => {
                         result.tag = .angle_bracket_angle_bracket_left;
@@ -2015,6 +2015,12 @@ test "tokenizer - invalid token with unfinished escape right before eof" {
     try testTokenize("'\\u", &.{.invalid});
 }
 
+test "tokenizer - saturating" {
+    try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
+    try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});
+    try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});
+}
+
 fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {
diff --git a/src/Air.zig b/src/Air.zig
@@ -44,7 +44,7 @@ pub const Inst = struct {
         /// is the same as both operands.
         /// Uses the `bin_op` field.
         addwrap,
-        /// Saturating integer addition. 
+        /// Saturating integer addition.
         /// Both operands are guaranteed to be the same type, and the result type
         /// is the same as both operands.
         /// Uses the `bin_op` field.
@@ -59,7 +59,7 @@ pub const Inst = struct {
         /// is the same as both operands.
         /// Uses the `bin_op` field.
         subwrap,
-        /// Saturating integer subtraction. 
+        /// Saturating integer subtraction.
         /// Both operands are guaranteed to be the same type, and the result type
         /// is the same as both operands.
         /// Uses the `bin_op` field.
@@ -74,7 +74,7 @@ pub const Inst = struct {
         /// is the same as both operands.
         /// Uses the `bin_op` field.
         mulwrap,
-        /// Saturating integer multiplication. 
+        /// Saturating integer multiplication.
         /// Both operands are guaranteed to be the same type, and the result type
         /// is the same as both operands.
         /// Uses the `bin_op` field.
diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp
@@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) {
                         t.state = TokenizeState_start;
                         break;
                     case '|':
-                        // t.out->ids.last() = TokenIdBitShiftLeftPipe;
                         t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;
                         break;
                     default: