zig

commit 22e38ffb54775c2e523665ac739f2a895d8757ad (tree)
parent a2330d0ea366e0ced7fa917d3bf1ccd022e932c3
Author: Jimmi Holst Christensen <jhc@liab.dk>
Date:   Tue,  3 Apr 2018 11:18:18 +0200

std.zig.tokenizer fixed tokens having wrong column and line

Diffstat:
M
std/zig/tokenizer.zig
 | 
21
++++++++++++---------

1 file changed, 12 insertions(+), 9 deletions(-)
diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig
@@ -271,6 +271,7 @@ pub const Tokenizer = struct {
             self.pending_invalid_token = null;
             return token;
         }
+        const start_index = self.index;
         var state = State.Start;
         var result = Token {
             .id = Token.Id.Eof,
@@ -279,7 +280,7 @@ pub const Tokenizer = struct {
             .line = self.line,
             .column = self.column,
         };
-        while (self.index < self.buffer.len) {
+        while (self.index < self.buffer.len) : (self.index += 1) {
             const c = self.buffer[self.index];
             switch (state) {
                 State.Start => switch (c) {
@@ -877,14 +878,6 @@ pub const Tokenizer = struct {
                     else => break,
                 },
             }
-
-            self.index += 1;
-            if (c == '\n') {
-                self.line += 1;
-                self.column = 0;
-            } else {
-                self.column += 1;
-            }
         } else if (self.index == self.buffer.len) {
             switch (state) {
                 State.Start,
@@ -983,6 +976,16 @@ pub const Tokenizer = struct {
                 },
             }
         }
+
+        for (self.buffer[start_index..self.index]) |c| {
+            if (c == '\n') {
+                self.line += 1;
+                self.column = 0;
+            } else {
+                self.column += 1;
+            }
+        }
+
         if (result.id == Token.Id.Eof) {
             if (self.pending_invalid_token) |token| {
                 self.pending_invalid_token = null;