zig

commit 0676c04681ad7104faa7634c781df386e839d17b (tree)
parent 943176bbfcf3125451fa64e082ee357c28e70413
Author: Igor Stojković <stojkovic.igor@gmail.com>
Date:   Mon, 23 Sep 2024 22:06:30 +0200

tokenizer: fix 0 byte following invalid (#21482)

closes #21481
Diffstat:
M
lib/std/zig/tokenizer.zig
 | 
3
+++

1 file changed, 3 insertions(+), 0 deletions(-)
diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
@@ -535,6 +535,8 @@ pub const Tokenizer = struct {
                 switch (self.buffer[self.index]) {
                     0 => if (self.index == self.buffer.len) {
                         result.tag = .invalid;
+                    } else {
+                        continue :state .invalid;
                     },
                     '\n' => result.tag = .invalid,
                     else => continue :state .invalid,
@@ -1250,6 +1252,7 @@ test "invalid token characters" {
 
 test "invalid literal/comment characters" {
     try testTokenize("\"\x00\"", &.{.invalid});
+    try testTokenize("`\x00`", &.{.invalid});
     try testTokenize("//\x00", &.{.invalid});
     try testTokenize("//\x1f", &.{.invalid});
     try testTokenize("//\x7f", &.{.invalid});