motiejus/zig
1
Fork 0
Code Releases Activity

tokenizer: detect null as non-first byte of a line comment

Browse Source 
Line comments do not produce actual tokens so they need special
handling for null bytes.

Closes #14346
This commit is contained in:
Veikka Tuominen
2023-01-17 20:39:19 +02:00
parent fc066992d9
commit 841b38aae8
1 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lib/std/zig/tokenizer.zig
View File

@@ -1151,7 +1151,13 @@ pub const Tokenizer = struct {
},
},
.line_comment => switch (c) {
0 => break,
0 => {
if (self.index != self.buffer.len) {
result.tag = .invalid;
self.index += 1;
}
break;
},
'\n' => {
state = .start;
result.loc.start = self.index + 1;
@@ -1865,6 +1871,9 @@ test "null byte before eof" {
try testTokenize("//\x00", &.{.invalid});
try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
try testTokenize("\x00", &.{.invalid});
try testTokenize("// NUL\x00\n", &.{.invalid});
try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {