zig0/tokenizer.h

#ifndef _ZIG1_TOKENIZER_H__
#define _ZIG1_TOKENIZER_H__

#include <stdbool.h>
#include <stdint.h>

typedef enum {
    TOKENIZER_TAG_INVALID,
    TOKENIZER_TAG_INVALID_PERIODASTERISKS,
    TOKENIZER_TAG_IDENTIFIER,
    TOKENIZER_TAG_STRING_LITERAL,
    TOKENIZER_TAG_MULTILINE_STRING_LITERAL_LINE,
    TOKENIZER_TAG_CHAR_LITERAL,
    TOKENIZER_TAG_EOF,
    TOKENIZER_TAG_BUILTIN,
    TOKENIZER_TAG_BANG,
    TOKENIZER_TAG_PIPE,
    TOKENIZER_TAG_PIPE_PIPE,
    TOKENIZER_TAG_PIPE_EQUAL,
    TOKENIZER_TAG_EQUAL,
    TOKENIZER_TAG_EQUAL_EQUAL,
    TOKENIZER_TAG_EQUAL_ANGLE_BRACKET_RIGHT,
    TOKENIZER_TAG_BANG_EQUAL,
    TOKENIZER_TAG_L_PAREN,
    TOKENIZER_TAG_R_PAREN,
    TOKENIZER_TAG_SEMICOLON,
    TOKENIZER_TAG_PERCENT,
    TOKENIZER_TAG_PERCENT_EQUAL,
    TOKENIZER_TAG_L_BRACE,
    TOKENIZER_TAG_R_BRACE,
    TOKENIZER_TAG_L_BRACKET,
    TOKENIZER_TAG_R_BRACKET,
    TOKENIZER_TAG_PERIOD,
    TOKENIZER_TAG_PERIOD_ASTERISK,
    TOKENIZER_TAG_ELLIPSIS2,
    TOKENIZER_TAG_ELLIPSIS3,
    TOKENIZER_TAG_CARET,
    TOKENIZER_TAG_CARET_EQUAL,
    TOKENIZER_TAG_PLUS,
    TOKENIZER_TAG_PLUS_PLUS,
    TOKENIZER_TAG_PLUS_EQUAL,
    TOKENIZER_TAG_PLUS_PERCENT,
    TOKENIZER_TAG_PLUS_PERCENT_EQUAL,
    TOKENIZER_TAG_PLUS_PIPE,
    TOKENIZER_TAG_PLUS_PIPE_EQUAL,
    TOKENIZER_TAG_MINUS,
    TOKENIZER_TAG_MINUS_EQUAL,
    TOKENIZER_TAG_MINUS_PERCENT,
    TOKENIZER_TAG_MINUS_PERCENT_EQUAL,
    TOKENIZER_TAG_MINUS_PIPE,
    TOKENIZER_TAG_MINUS_PIPE_EQUAL,
    TOKENIZER_TAG_ASTERISK,
    TOKENIZER_TAG_ASTERISK_EQUAL,
    TOKENIZER_TAG_ASTERISK_ASTERISK,
    TOKENIZER_TAG_ASTERISK_PERCENT,
    TOKENIZER_TAG_ASTERISK_PERCENT_EQUAL,
    TOKENIZER_TAG_ASTERISK_PIPE,
    TOKENIZER_TAG_ASTERISK_PIPE_EQUAL,
    TOKENIZER_TAG_ARROW,
    TOKENIZER_TAG_COLON,
    TOKENIZER_TAG_SLASH,
    TOKENIZER_TAG_SLASH_EQUAL,
    TOKENIZER_TAG_COMMA,
    TOKENIZER_TAG_AMPERSAND,
    TOKENIZER_TAG_AMPERSAND_EQUAL,
    TOKENIZER_TAG_QUESTION_MARK,
    TOKENIZER_TAG_ANGLE_BRACKET_LEFT,
    TOKENIZER_TAG_ANGLE_BRACKET_LEFT_EQUAL,
    TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT,
    TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_EQUAL,
    TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE,
    TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE_EQUAL,
    TOKENIZER_TAG_ANGLE_BRACKET_RIGHT,
    TOKENIZER_TAG_ANGLE_BRACKET_RIGHT_EQUAL,
    TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT,
    TOKENIZER_TAG_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT_EQUAL,
    TOKENIZER_TAG_TILDE,
    TOKENIZER_TAG_NUMBER_LITERAL,
    TOKENIZER_TAG_DOC_COMMENT,
    TOKENIZER_TAG_CONTAINER_DOC_COMMENT,
    TOKENIZER_TAG_KEYWORD_ADDRSPACE,
    TOKENIZER_TAG_KEYWORD_ALIGN,
    TOKENIZER_TAG_KEYWORD_ALLOWZERO,
    TOKENIZER_TAG_KEYWORD_AND,
    TOKENIZER_TAG_KEYWORD_ANYFRAME,
    TOKENIZER_TAG_KEYWORD_ANYTYPE,
    TOKENIZER_TAG_KEYWORD_ASM,
    TOKENIZER_TAG_KEYWORD_ASYNC,
    TOKENIZER_TAG_KEYWORD_AWAIT,
    TOKENIZER_TAG_KEYWORD_BREAK,
    TOKENIZER_TAG_KEYWORD_CALLCONV,
    TOKENIZER_TAG_KEYWORD_CATCH,
    TOKENIZER_TAG_KEYWORD_COMPTIME,
    TOKENIZER_TAG_KEYWORD_CONST,
    TOKENIZER_TAG_KEYWORD_CONTINUE,
    TOKENIZER_TAG_KEYWORD_DEFER,
    TOKENIZER_TAG_KEYWORD_ELSE,
    TOKENIZER_TAG_KEYWORD_ENUM,
    TOKENIZER_TAG_KEYWORD_ERRDEFER,
    TOKENIZER_TAG_KEYWORD_ERROR,
    TOKENIZER_TAG_KEYWORD_EXPORT,
    TOKENIZER_TAG_KEYWORD_EXTERN,
    TOKENIZER_TAG_KEYWORD_FN,
    TOKENIZER_TAG_KEYWORD_FOR,
    TOKENIZER_TAG_KEYWORD_IF,
    TOKENIZER_TAG_KEYWORD_INLINE,
    TOKENIZER_TAG_KEYWORD_NOALIAS,
    TOKENIZER_TAG_KEYWORD_NOINLINE,
    TOKENIZER_TAG_KEYWORD_NOSUSPEND,
    TOKENIZER_TAG_KEYWORD_OPAQUE,
    TOKENIZER_TAG_KEYWORD_OR,
    TOKENIZER_TAG_KEYWORD_ORELSE,
    TOKENIZER_TAG_KEYWORD_PACKED,
    TOKENIZER_TAG_KEYWORD_PUB,
    TOKENIZER_TAG_KEYWORD_RESUME,
    TOKENIZER_TAG_KEYWORD_RETURN,
    TOKENIZER_TAG_KEYWORD_LINKSECTION,
    TOKENIZER_TAG_KEYWORD_STRUCT,
    TOKENIZER_TAG_KEYWORD_SUSPEND,
    TOKENIZER_TAG_KEYWORD_SWITCH,
    TOKENIZER_TAG_KEYWORD_TEST,
    TOKENIZER_TAG_KEYWORD_THREADLOCAL,
    TOKENIZER_TAG_KEYWORD_TRY,
    TOKENIZER_TAG_KEYWORD_UNION,
    TOKENIZER_TAG_KEYWORD_UNREACHABLE,
    TOKENIZER_TAG_KEYWORD_USINGNAMESPACE,
    TOKENIZER_TAG_KEYWORD_VAR,
    TOKENIZER_TAG_KEYWORD_VOLATILE,
    TOKENIZER_TAG_KEYWORD_WHILE,
} tokenizer_tag;

typedef enum {
    TOKENIZER_STATE_START,
    TOKENIZER_STATE_EXPECT_NEWLINE,
    TOKENIZER_STATE_IDENTIFIER,
    TOKENIZER_STATE_BUILTIN,
    TOKENIZER_STATE_STRING_LITERAL,
    TOKENIZER_STATE_STRING_LITERAL_BACKSLASH,
    TOKENIZER_STATE_MULTILINE_STRING_LITERAL_LINE,
    TOKENIZER_STATE_CHAR_LITERAL,
    TOKENIZER_STATE_CHAR_LITERAL_BACKSLASH,
    TOKENIZER_STATE_BACKSLASH,
    TOKENIZER_STATE_EQUAL,
    TOKENIZER_STATE_BANG,
    TOKENIZER_STATE_PIPE,
    TOKENIZER_STATE_MINUS,
    TOKENIZER_STATE_MINUS_PERCENT,
    TOKENIZER_STATE_MINUS_PIPE,
    TOKENIZER_STATE_ASTERISK,
    TOKENIZER_STATE_ASTERISK_PERCENT,
    TOKENIZER_STATE_ASTERISK_PIPE,
    TOKENIZER_STATE_SLASH,
    TOKENIZER_STATE_LINE_COMMENT_START,
    TOKENIZER_STATE_LINE_COMMENT,
    TOKENIZER_STATE_DOC_COMMENT_START,
    TOKENIZER_STATE_DOC_COMMENT,
    TOKENIZER_STATE_INT,
    TOKENIZER_STATE_INT_EXPONENT,
    TOKENIZER_STATE_INT_PERIOD,
    TOKENIZER_STATE_FLOAT,
    TOKENIZER_STATE_FLOAT_EXPONENT,
    TOKENIZER_STATE_AMPERSAND,
    TOKENIZER_STATE_CARET,
    TOKENIZER_STATE_PERCENT,
    TOKENIZER_STATE_PLUS,
    TOKENIZER_STATE_PLUS_PERCENT,
    TOKENIZER_STATE_PLUS_PIPE,
    TOKENIZER_STATE_ANGLE_BRACKET_LEFT,
    TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT,
    TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_LEFT_PIPE,
    TOKENIZER_STATE_ANGLE_BRACKET_RIGHT,
    TOKENIZER_STATE_ANGLE_BRACKET_ANGLE_BRACKET_RIGHT,
    TOKENIZER_STATE_PERIOD,
    TOKENIZER_STATE_PERIOD_2,
    TOKENIZER_STATE_PERIOD_ASTERISK,
    TOKENIZER_STATE_SAW_AT_SIGN,
    TOKENIZER_STATE_INVALID,
} tokenizer_state;

typedef struct {
    tokenizer_tag tag;
    struct {
        uint32_t start, end;
    } loc;
} tokenizer_token;

typedef struct {
    const char* buffer;
    const uint32_t buffer_len;
    uint32_t index;
} tokenizer;

tokenizer tokenizer_init(const char* buffer, uint32_t len);
tokenizer_token tokenizer_next(tokenizer* self);

#endif