#ifndef _ZIG1_TOKENIZER_H__ #define _ZIG1_TOKENIZER_H__ #include #include typedef enum { TOKENIZER_TAG_INVALID, TOKENIZER_TAG_INVALID_PERIODASTERISKS, TOKENIZER_TAG_IDENTIFIER, TOKENIZER_TAG_STRING_LITERAL, TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE, TOKENIZER_TAG_CHAR_LITERAL, TOKENIZER_TAG_EOF, TOKENIZER_TAG_BUILTIN, TOKENIZER_TAG_BANG, TOKENIZER_TAG_PIPE, TOKENIZER_TAG_PIPE_PIPE, TOKENIZER_TAG_PIPE_EQUAL, TOKENIZER_TAG_EQUAL, TOKENIZER_TAG_EQUAL_EQUAL, TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT, TOKENIZER_TAG_BANG_EQUAL, TOKENIZER_TAG_L_PAREN, TOKENIZER_TAG_R_PAREN, TOKENIZER_TAG_SEMICOLON, TOKENIZER_TAG_PERCENT, TOKENIZER_TAG_PERCENT_EQUAL, TOKENIZER_TAG_L_BRACE, TOKENIZER_TAG_R_BRACE, TOKENIZER_TAG_L_BRACKET, TOKENIZER_TAG_R_BRACKET, TOKENIZER_TAG_PERIOD, TOKENIZER_TAG_PERIOD_ASTERISK, TOKENIZER_TAG_ELLIPSIS2, TOKENIZER_TAG_ELLIPSIS3, TOKENIZER_TAG_CARET, TOKENIZER_TAG_CARET_EQUAL, TOKENIZER_TAG_PLUS, TOKENIZER_TAG_PLUS_PLUS, TOKENIZER_TAG_PLUS_EQUAL, TOKENIZER_TAG_PLUS_PERCENT, TOKENIZER_TAG_PLUS_PERCENT_EQUAL, TOKENIZER_TAG_PLUS_PIPE, TOKENIZER_TAG_PLUS_PIPE_EQUAL, TOKENIZER_TAG_MINUS, TOKENIZER_TAG_MINUS_EQUAL, TOKENIZER_TAG_MINUS_PERCENT, TOKENIZER_TAG_MINUS_PERCENT_EQUAL, TOKENIZER_TAG_MINUS_PIPE, TOKENIZER_TAG_MINUS_PIPE_EQUAL, TOKENIZER_TAG_ASTERISK, TOKENIZER_TAG_ASTERISK_EQUAL, TOKENIZER_TAG_ASTERISK_ASTERISK, TOKENIZER_TAG_ASTERISK_PERCENT, TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL, TOKENIZER_TAG_ASTERISK_PIPE, TOKENIZER_TAG_ASTERISK_PIPE_EQUAL, TOKENIZER_TAG_ARROW, TOKENIZER_TAG_COLON, TOKENIZER_TAG_SLASH, TOKENIZER_TAG_SLASH_EQUAL, TOKENIZER_TAG_COMMA, TOKENIZER_TAG_AMPERSAND, TOKENIZER_TAG_AMPERSAND_EQUAL, TOKENIZER_TAG_QUESTION_MARK, TOKENIZER_TAG_ANGLE_BRACKET_LEFT, TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL, TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT, TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL, TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE, TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL, TOKENIZER_TAG_ANGLE_BRACKET_RIGHT, TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL, TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT, TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL, TOKENIZER_TAG_TILDE, TOKENIZER_TAG_NUMBER_LITERAL, TOKENIZER_TAG_DOC_COMMENT, TOKENIZER_TAG_CONTAINER_DOC_COMMENT, TOKENIZER_TAG_KEYWORD_ADDRSPACE, TOKENIZER_TAG_KEYWORD_ALIGN, TOKENIZER_TAG_KEYWORD_ALLOWZERO, TOKENIZER_TAG_KEYWORD_AND, TOKENIZER_TAG_KEYWORD_ANYFRAME, TOKENIZER_TAG_KEYWORD_ANYTYPE, TOKENIZER_TAG_KEYWORD_ASM, TOKENIZER_TAG_KEYWORD_ASYNC, TOKENIZER_TAG_KEYWORD_AWAIT, TOKENIZER_TAG_KEYWORD_BREAK, TOKENIZER_TAG_KEYWORD_CALLCONV, TOKENIZER_TAG_KEYWORD_CATCH, TOKENIZER_TAG_KEYWORD_COMPTIME, TOKENIZER_TAG_KEYWORD_CONST, TOKENIZER_TAG_KEYWORD_CONTINUE, TOKENIZER_TAG_KEYWORD_DEFER, TOKENIZER_TAG_KEYWORD_ELSE, TOKENIZER_TAG_KEYWORD_ENUM, TOKENIZER_TAG_KEYWORD_ERRDEFER, TOKENIZER_TAG_KEYWORD_ERROR, TOKENIZER_TAG_KEYWORD_EXPORT, TOKENIZER_TAG_KEYWORD_EXTERN, TOKENIZER_TAG_KEYWORD_FN, TOKENIZER_TAG_KEYWORD_FOR, TOKENIZER_TAG_KEYWORD_IF, TOKENIZER_TAG_KEYWORD_INLINE, TOKENIZER_TAG_KEYWORD_NOALIAS, TOKENIZER_TAG_KEYWORD_NOINLINE, TOKENIZER_TAG_KEYWORD_NOSUSPEND, TOKENIZER_TAG_KEYWORD_OPAQUE, TOKENIZER_TAG_KEYWORD_OR, TOKENIZER_TAG_KEYWORD_ORELSE, TOKENIZER_TAG_KEYWORD_PACKED, TOKENIZER_TAG_KEYWORD_PUB, TOKENIZER_TAG_KEYWORD_RESUME, TOKENIZER_TAG_KEYWORD_RETURN, TOKENIZER_TAG_KEYWORD_LINKSECTION, TOKENIZER_TAG_KEYWORD_STRUCT, TOKENIZER_TAG_KEYWORD_SUSPEND, TOKENIZER_TAG_KEYWORD_SWITCH, TOKENIZER_TAG_KEYWORD_TEST, TOKENIZER_TAG_KEYWORD_THREADLOCAL, TOKENIZER_TAG_KEYWORD_TRY, TOKENIZER_TAG_KEYWORD_UNION, TOKENIZER_TAG_KEYWORD_UNREACHABLE, TOKENIZER_TAG_KEYWORD_USINGNAMESPACE, TOKENIZER_TAG_KEYWORD_VAR, TOKENIZER_TAG_KEYWORD_VOLATILE, TOKENIZER_TAG_KEYWORD_WHILE, } tokenizer_tag; typedef enum { TOKENIZER_STATE_START, TOKENIZER_STATE_EXPECT_NEWLINE, TOKENIZER_STATE_IDENTIFIER, TOKENIZER_STATE_BUILTIN, TOKENIZER_STATE_STRING_LITERAL, TOKENIZER_STATE_STRING_LITERAL_BACKSLASH, TOKENIZER_STATE_MULTILINE_STRING_LITERAL_LINE, TOKENIZER_STATE_CHAR_LITERAL, TOKENIZER_STATE_CHAR_LITERAL_BACKSLASH, TOKENIZER_STATE_BACKSLASH, TOKENIZER_STATE_EQUAL, TOKENIZER_STATE_BANG, TOKENIZER_STATE_PIPE, TOKENIZER_STATE_MINUS, TOKENIZER_STATE_MINUS_PERCENT, TOKENIZER_STATE_MINUS_PIPE, TOKENIZER_STATE_ASTERISK, TOKENIZER_STATE_ASTERISK_PERCENT, TOKENIZER_STATE_ASTERISK_PIPE, TOKENIZER_STATE_SLASH, TOKENIZER_STATE_LINE_COMMENT_START, TOKENIZER_STATE_LINE_COMMENT, TOKENIZER_STATE_DOC_COMMENT_START, TOKENIZER_STATE_DOC_COMMENT, TOKENIZER_STATE_INT, TOKENIZER_STATE_INT_EXPONENT, TOKENIZER_STATE_INT_PERIOD, TOKENIZER_STATE_FLOAT, TOKENIZER_STATE_FLOAT_EXPONENT, TOKENIZER_STATE_AMPERSAND, TOKENIZER_STATE_CARET, TOKENIZER_STATE_PERCENT, TOKENIZER_STATE_PLUS, TOKENIZER_STATE_PLUS_PERCENT, TOKENIZER_STATE_PLUS_PIPE, TOKENIZER_STATE_ANGLE_BRACKET_LEFT, TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT, TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE, TOKENIZER_STATE_ANGLE_BRACKET_RIGHT, TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT, TOKENIZER_STATE_PERIOD, TOKENIZER_STATE_PERIOD_2, TOKENIZER_STATE_PERIOD_ASTERISK, TOKENIZER_STATE_SAW_AT_SIGN, TOKENIZER_STATE_INVALID, } tokenizer_state; typedef struct { tokenizer_tag tag; struct { uint32_t start, end; } loc; } tokenizer_token; typedef struct { const char* buffer; const uint32_t buffer_len; uint32_t index; } tokenizer; tokenizer tokenizer_init(const char* buffer, uint32_t len); tokenizer_token tokenizer_next(tokenizer* self); #endif