motiejus/zig0
1
Fork 0
Code Releases Activity

start ast + fix type names in tokenizer

Browse Source
This commit is contained in:
Motiejus Jakštys
2024-12-15 00:04:23 +02:00
parent 6863e34fbc
commit c2915d2eaa
6 changed files with 1135 additions and 587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

348
tokenizer.h
View File

@@ -1,188 +1,188 @@
#ifndef __ZIG1_TOKENIZER_H__
#define __ZIG1_TOKENIZER_H__
#ifndef _ZIG1_TOKENIZER_H__
#define _ZIG1_TOKENIZER_H__
#include <stdbool.h>
#include <stdint.h>
typedef enum {
TOKEN_TAG_INVALID,
TOKEN_TAG_INVALID_PERIODASTERISKS,
TOKEN_TAG_IDENTIFIER,
TOKEN_TAG_STRING_LITERAL,
TOKEN_TAG_MULTILINE_STRING_LITERAL_LINE,
TOKEN_TAG_CHAR_LITERAL,
TOKEN_TAG_EOF,
TOKEN_TAG_BUILTIN,
TOKEN_TAG_BANG,
TOKEN_TAG_PIPE,
TOKEN_TAG_PIPE_PIPE,
TOKEN_TAG_PIPE_EQUAL,
TOKEN_TAG_EQUAL,
TOKEN_TAG_EQUAL_EQUAL,
TOKEN_TAG_EQUAL_ANGLE_BRACKET_RIGHT,
TOKEN_TAG_BANG_EQUAL,
TOKEN_TAG_L_PAREN,
TOKEN_TAG_R_PAREN,
TOKEN_TAG_SEMICOLON,
TOKEN_TAG_PERCENT,
TOKEN_TAG_PERCENT_EQUAL,
TOKEN_TAG_L_BRACE,
TOKEN_TAG_R_BRACE,
TOKEN_TAG_L_BRACKET,
TOKEN_TAG_R_BRACKET,
TOKEN_TAG_PERIOD,
TOKEN_TAG_PERIOD_ASTERISK,
TOKEN_TAG_ELLIPSIS2,
TOKEN_TAG_ELLIPSIS3,
TOKEN_TAG_CARET,
TOKEN_TAG_CARET_EQUAL,
TOKEN_TAG_PLUS,
TOKEN_TAG_PLUS_PLUS,
TOKEN_TAG_PLUS_EQUAL,
TOKEN_TAG_PLUS_PERCENT,
TOKEN_TAG_PLUS_PERCENT_EQUAL,
TOKEN_TAG_PLUS_PIPE,
TOKEN_TAG_PLUS_PIPE_EQUAL,
TOKEN_TAG_MINUS,
TOKEN_TAG_MINUS_EQUAL,
TOKEN_TAG_MINUS_PERCENT,
TOKEN_TAG_MINUS_PERCENT_EQUAL,
TOKEN_TAG_MINUS_PIPE,
TOKEN_TAG_MINUS_PIPE_EQUAL,
TOKEN_TAG_ASTERISK,
TOKEN_TAG_ASTERISK_EQUAL,
TOKEN_TAG_ASTERISK_ASTERISK,
TOKEN_TAG_ASTERISK_PERCENT,
TOKEN_TAG_ASTERISK_PERCENT_EQUAL,
TOKEN_TAG_ASTERISK_PIPE,
TOKEN_TAG_ASTERISK_PIPE_EQUAL,
TOKEN_TAG_ARROW,
TOKEN_TAG_COLON,
TOKEN_TAG_SLASH,
TOKEN_TAG_SLASH_EQUAL,
TOKEN_TAG_COMMA,
TOKEN_TAG_AMPERSAND,
TOKEN_TAG_AMPERSAND_EQUAL,
TOKEN_TAG_QUESTION_MARK,
TOKEN_TAG_ANGLE_BRACKET_LEFT,
TOKEN_TAG_ANGLE_BRACKET_LEFT_EQUAL,
TOKEN_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT,
TOKEN_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL,
TOKEN_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE,
TOKEN_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL,
TOKEN_TAG_ANGLE_BRACKET_RIGHT,
TOKEN_TAG_ANGLE_BRACKET_RIGHT_EQUAL,
TOKEN_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT,
TOKEN_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL,
TOKEN_TAG_TILDE,
TOKEN_TAG_NUMBER_LITERAL,
TOKEN_TAG_DOC_COMMENT,
TOKEN_TAG_CONTAINER_DOC_COMMENT,
TOKEN_TAG_KEYWORD_ADDRSPACE,
TOKEN_TAG_KEYWORD_ALIGN,
TOKEN_TAG_KEYWORD_ALLOWZERO,
TOKEN_TAG_KEYWORD_AND,
TOKEN_TAG_KEYWORD_ANYFRAME,
TOKEN_TAG_KEYWORD_ANYTYPE,
TOKEN_TAG_KEYWORD_ASM,
TOKEN_TAG_KEYWORD_ASYNC,
TOKEN_TAG_KEYWORD_AWAIT,
TOKEN_TAG_KEYWORD_BREAK,
TOKEN_TAG_KEYWORD_CALLCONV,
TOKEN_TAG_KEYWORD_CATCH,
TOKEN_TAG_KEYWORD_COMPTIME,
TOKEN_TAG_KEYWORD_CONST,
TOKEN_TAG_KEYWORD_CONTINUE,
TOKEN_TAG_KEYWORD_DEFER,
TOKEN_TAG_KEYWORD_ELSE,
TOKEN_TAG_KEYWORD_ENUM,
TOKEN_TAG_KEYWORD_ERRDEFER,
TOKEN_TAG_KEYWORD_ERROR,
TOKEN_TAG_KEYWORD_EXPORT,
TOKEN_TAG_KEYWORD_EXTERN,
TOKEN_TAG_KEYWORD_FN,
TOKEN_TAG_KEYWORD_FOR,
TOKEN_TAG_KEYWORD_IF,
TOKEN_TAG_KEYWORD_INLINE,
TOKEN_TAG_KEYWORD_NOALIAS,
TOKEN_TAG_KEYWORD_NOINLINE,
TOKEN_TAG_KEYWORD_NOSUSPEND,
TOKEN_TAG_KEYWORD_OPAQUE,
TOKEN_TAG_KEYWORD_OR,
TOKEN_TAG_KEYWORD_ORELSE,
TOKEN_TAG_KEYWORD_PACKED,
TOKEN_TAG_KEYWORD_PUB,
TOKEN_TAG_KEYWORD_RESUME,
TOKEN_TAG_KEYWORD_RETURN,
TOKEN_TAG_KEYWORD_LINKSECTION,
TOKEN_TAG_KEYWORD_STRUCT,
TOKEN_TAG_KEYWORD_SUSPEND,
TOKEN_TAG_KEYWORD_SWITCH,
TOKEN_TAG_KEYWORD_TEST,
TOKEN_TAG_KEYWORD_THREADLOCAL,
TOKEN_TAG_KEYWORD_TRY,
TOKEN_TAG_KEYWORD_UNION,
TOKEN_TAG_KEYWORD_UNREACHABLE,
TOKEN_TAG_KEYWORD_USINGNAMESPACE,
TOKEN_TAG_KEYWORD_VAR,
TOKEN_TAG_KEYWORD_VOLATILE,
TOKEN_TAG_KEYWORD_WHILE,
} token_tag;
TOKENIZER_TAG_INVALID,
TOKENIZER_TAG_INVALID_PERIODASTERISKS,
TOKENIZER_TAG_IDENTIFIER,
TOKENIZER_TAG_STRING_LITERAL,
TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE,
TOKENIZER_TAG_CHAR_LITERAL,
TOKENIZER_TAG_EOF,
TOKENIZER_TAG_BUILTIN,
TOKENIZER_TAG_BANG,
TOKENIZER_TAG_PIPE,
TOKENIZER_TAG_PIPE_PIPE,
TOKENIZER_TAG_PIPE_EQUAL,
TOKENIZER_TAG_EQUAL,
TOKENIZER_TAG_EQUAL_EQUAL,
TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT,
TOKENIZER_TAG_BANG_EQUAL,
TOKENIZER_TAG_L_PAREN,
TOKENIZER_TAG_R_PAREN,
TOKENIZER_TAG_SEMICOLON,
TOKENIZER_TAG_PERCENT,
TOKENIZER_TAG_PERCENT_EQUAL,
TOKENIZER_TAG_L_BRACE,
TOKENIZER_TAG_R_BRACE,
TOKENIZER_TAG_L_BRACKET,
TOKENIZER_TAG_R_BRACKET,
TOKENIZER_TAG_PERIOD,
TOKENIZER_TAG_PERIOD_ASTERISK,
TOKENIZER_TAG_ELLIPSIS2,
TOKENIZER_TAG_ELLIPSIS3,
TOKENIZER_TAG_CARET,
TOKENIZER_TAG_CARET_EQUAL,
TOKENIZER_TAG_PLUS,
TOKENIZER_TAG_PLUS_PLUS,
TOKENIZER_TAG_PLUS_EQUAL,
TOKENIZER_TAG_PLUS_PERCENT,
TOKENIZER_TAG_PLUS_PERCENT_EQUAL,
TOKENIZER_TAG_PLUS_PIPE,
TOKENIZER_TAG_PLUS_PIPE_EQUAL,
TOKENIZER_TAG_MINUS,
TOKENIZER_TAG_MINUS_EQUAL,
TOKENIZER_TAG_MINUS_PERCENT,
TOKENIZER_TAG_MINUS_PERCENT_EQUAL,
TOKENIZER_TAG_MINUS_PIPE,
TOKENIZER_TAG_MINUS_PIPE_EQUAL,
TOKENIZER_TAG_ASTERISK,
TOKENIZER_TAG_ASTERISK_EQUAL,
TOKENIZER_TAG_ASTERISK_ASTERISK,
TOKENIZER_TAG_ASTERISK_PERCENT,
TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL,
TOKENIZER_TAG_ASTERISK_PIPE,
TOKENIZER_TAG_ASTERISK_PIPE_EQUAL,
TOKENIZER_TAG_ARROW,
TOKENIZER_TAG_COLON,
TOKENIZER_TAG_SLASH,
TOKENIZER_TAG_SLASH_EQUAL,
TOKENIZER_TAG_COMMA,
TOKENIZER_TAG_AMPERSAND,
TOKENIZER_TAG_AMPERSAND_EQUAL,
TOKENIZER_TAG_QUESTION_MARK,
TOKENIZER_TAG_ANGLE_BRACKET_LEFT,
TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL,
TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT,
TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL,
TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE,
TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL,
TOKENIZER_TAG_ANGLE_BRACKET_RIGHT,
TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL,
TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT,
TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL,
TOKENIZER_TAG_TILDE,
TOKENIZER_TAG_NUMBER_LITERAL,
TOKENIZER_TAG_DOC_COMMENT,
TOKENIZER_TAG_CONTAINER_DOC_COMMENT,
TOKENIZER_TAG_KEYWORD_ADDRSPACE,
TOKENIZER_TAG_KEYWORD_ALIGN,
TOKENIZER_TAG_KEYWORD_ALLOWZERO,
TOKENIZER_TAG_KEYWORD_AND,
TOKENIZER_TAG_KEYWORD_ANYFRAME,
TOKENIZER_TAG_KEYWORD_ANYTYPE,
TOKENIZER_TAG_KEYWORD_ASM,
TOKENIZER_TAG_KEYWORD_ASYNC,
TOKENIZER_TAG_KEYWORD_AWAIT,
TOKENIZER_TAG_KEYWORD_BREAK,
TOKENIZER_TAG_KEYWORD_CALLCONV,
TOKENIZER_TAG_KEYWORD_CATCH,
TOKENIZER_TAG_KEYWORD_COMPTIME,
TOKENIZER_TAG_KEYWORD_CONST,
TOKENIZER_TAG_KEYWORD_CONTINUE,
TOKENIZER_TAG_KEYWORD_DEFER,
TOKENIZER_TAG_KEYWORD_ELSE,
TOKENIZER_TAG_KEYWORD_ENUM,
TOKENIZER_TAG_KEYWORD_ERRDEFER,
TOKENIZER_TAG_KEYWORD_ERROR,
TOKENIZER_TAG_KEYWORD_EXPORT,
TOKENIZER_TAG_KEYWORD_EXTERN,
TOKENIZER_TAG_KEYWORD_FN,
TOKENIZER_TAG_KEYWORD_FOR,
TOKENIZER_TAG_KEYWORD_IF,
TOKENIZER_TAG_KEYWORD_INLINE,
TOKENIZER_TAG_KEYWORD_NOALIAS,
TOKENIZER_TAG_KEYWORD_NOINLINE,
TOKENIZER_TAG_KEYWORD_NOSUSPEND,
TOKENIZER_TAG_KEYWORD_OPAQUE,
TOKENIZER_TAG_KEYWORD_OR,
TOKENIZER_TAG_KEYWORD_ORELSE,
TOKENIZER_TAG_KEYWORD_PACKED,
TOKENIZER_TAG_KEYWORD_PUB,
TOKENIZER_TAG_KEYWORD_RESUME,
TOKENIZER_TAG_KEYWORD_RETURN,
TOKENIZER_TAG_KEYWORD_LINKSECTION,
TOKENIZER_TAG_KEYWORD_STRUCT,
TOKENIZER_TAG_KEYWORD_SUSPEND,
TOKENIZER_TAG_KEYWORD_SWITCH,
TOKENIZER_TAG_KEYWORD_TEST,
TOKENIZER_TAG_KEYWORD_THREADLOCAL,
TOKENIZER_TAG_KEYWORD_TRY,
TOKENIZER_TAG_KEYWORD_UNION,
TOKENIZER_TAG_KEYWORD_UNREACHABLE,
TOKENIZER_TAG_KEYWORD_USINGNAMESPACE,
TOKENIZER_TAG_KEYWORD_VAR,
TOKENIZER_TAG_KEYWORD_VOLATILE,
TOKENIZER_TAG_KEYWORD_WHILE,
} tokenizer_tag;
typedef enum {
TOKEN_STATE_START,
TOKEN_STATE_EXPECT_NEWLINE,
TOKEN_STATE_IDENTIFIER,
TOKEN_STATE_BUILTIN,
TOKEN_STATE_STRING_LITERAL,
TOKEN_STATE_STRING_LITERAL_BACKSLASH,
TOKEN_STATE_MULTILINE_STRING_LITERAL_LINE,
TOKEN_STATE_CHAR_LITERAL,
TOKEN_STATE_CHAR_LITERAL_BACKSLASH,
TOKEN_STATE_BACKSLASH,
TOKEN_STATE_EQUAL,
TOKEN_STATE_BANG,
TOKEN_STATE_PIPE,
TOKEN_STATE_MINUS,
TOKEN_STATE_MINUS_PERCENT,
TOKEN_STATE_MINUS_PIPE,
TOKEN_STATE_ASTERISK,
TOKEN_STATE_ASTERISK_PERCENT,
TOKEN_STATE_ASTERISK_PIPE,
TOKEN_STATE_SLASH,
TOKEN_STATE_LINE_COMMENT_START,
TOKEN_STATE_LINE_COMMENT,
TOKEN_STATE_DOC_COMMENT_START,
TOKEN_STATE_DOC_COMMENT,
TOKEN_STATE_INT,
TOKEN_STATE_INT_EXPONENT,
TOKEN_STATE_INT_PERIOD,
TOKEN_STATE_FLOAT,
TOKEN_STATE_FLOAT_EXPONENT,
TOKEN_STATE_AMPERSAND,
TOKEN_STATE_CARET,
TOKEN_STATE_PERCENT,
TOKEN_STATE_PLUS,
TOKEN_STATE_PLUS_PERCENT,
TOKEN_STATE_PLUS_PIPE,
TOKEN_STATE_ANGLE_BRACKET_LEFT,
TOKEN_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT,
TOKEN_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE,
TOKEN_STATE_ANGLE_BRACKET_RIGHT,
TOKEN_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT,
TOKEN_STATE_PERIOD,
TOKEN_STATE_PERIOD_2,
TOKEN_STATE_PERIOD_ASTERISK,
TOKEN_STATE_SAW_AT_SIGN,
TOKEN_STATE_INVALID,
} token_state;
TOKENIZER_STATE_START,
TOKENIZER_STATE_EXPECT_NEWLINE,
TOKENIZER_STATE_IDENTIFIER,
TOKENIZER_STATE_BUILTIN,
TOKENIZER_STATE_STRING_LITERAL,
TOKENIZER_STATE_STRING_LITERAL_BACKSLASH,
TOKENIZER_STATE_MULTILINE_STRING_LITERAL_LINE,
TOKENIZER_STATE_CHAR_LITERAL,
TOKENIZER_STATE_CHAR_LITERAL_BACKSLASH,
TOKENIZER_STATE_BACKSLASH,
TOKENIZER_STATE_EQUAL,
TOKENIZER_STATE_BANG,
TOKENIZER_STATE_PIPE,
TOKENIZER_STATE_MINUS,
TOKENIZER_STATE_MINUS_PERCENT,
TOKENIZER_STATE_MINUS_PIPE,
TOKENIZER_STATE_ASTERISK,
TOKENIZER_STATE_ASTERISK_PERCENT,
TOKENIZER_STATE_ASTERISK_PIPE,
TOKENIZER_STATE_SLASH,
TOKENIZER_STATE_LINE_COMMENT_START,
TOKENIZER_STATE_LINE_COMMENT,
TOKENIZER_STATE_DOC_COMMENT_START,
TOKENIZER_STATE_DOC_COMMENT,
TOKENIZER_STATE_INT,
TOKENIZER_STATE_INT_EXPONENT,
TOKENIZER_STATE_INT_PERIOD,
TOKENIZER_STATE_FLOAT,
TOKENIZER_STATE_FLOAT_EXPONENT,
TOKENIZER_STATE_AMPERSAND,
TOKENIZER_STATE_CARET,
TOKENIZER_STATE_PERCENT,
TOKENIZER_STATE_PLUS,
TOKENIZER_STATE_PLUS_PERCENT,
TOKENIZER_STATE_PLUS_PIPE,
TOKENIZER_STATE_ANGLE_BRACKET_LEFT,
TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT,
TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE,
TOKENIZER_STATE_ANGLE_BRACKET_RIGHT,
TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT,
TOKENIZER_STATE_PERIOD,
TOKENIZER_STATE_PERIOD_2,
TOKENIZER_STATE_PERIOD_ASTERISK,
TOKENIZER_STATE_SAW_AT_SIGN,
TOKENIZER_STATE_INVALID,
} tokenizer_state;
typedef struct {
token_tag tag;
tokenizer_tag tag;
struct {
uint32_t start, end;
} loc;
} token;
} tokenizer_token;
typedef struct {
const char* buffer;
@@ -191,6 +191,6 @@ typedef struct {
} tokenizer;
tokenizer tokenizer_init(const char* buffer, uint32_t len);
token tokenizer_next(tokenizer* self);
tokenizer_token tokenizer_next(tokenizer* self);
#endif