motiejus/zig0
1
Fork 0
Code Releases Activity

more parser

Browse Source
This commit is contained in:
Motiejus Jakštys
2024-12-31 19:14:12 +02:00
parent 3264d1747e
commit 85dfbe9d09
4 changed files with 136 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
parser.c
View File

@@ -7,12 +7,13 @@
#include "parser.h"
const AstNodeIndex null_node = 0;
const AstTokenIndex null_token = ~(AstTokenIndex)(0);
typedef struct {
enum {
FIELD_STATE_NONE,
FIELD_STATE_SEEN,
FIELD_STATE_END // sets "end"
FIELD_STATE_END
} tag;
union {
uint32_t end;
@@ -30,6 +31,22 @@ typedef struct {
} payload;
} SmallSpan;
typedef struct {
AstNodeIndexSlice* scratch;
uint32_t old_len;
} CleanupScratch;
static CleanupScratch initCleanupScratch(Parser* p) {
return (CleanupScratch) {
.scratch = &p->scratch,
.old_len = p->scratch.len,
};
}
static void cleanupScratch(CleanupScratch* c) {
c->scratch->len = c->old_len;
}
static AstSubRange listToSpan(Parser* p, const AstNodeIndex* list, uint32_t count) {
SLICE_ENSURE_CAPACITY(AstNodeIndex, &p->extra_data, count);
memcpy(&p->extra_data.arr, list, count * sizeof(AstNodeIndex));
@@ -60,31 +77,16 @@ static AstTokenIndex nextToken(Parser* p) {
return p->tok_i++;
}
static AstTokenIndex eatToken(Parser* p, TokenizerTag tag, bool* ok) {
static AstTokenIndex eatToken(Parser* p, TokenizerTag tag) {
if (p->token_tags[p->tok_i] == tag) {
if (ok != NULL)
*ok = true;
return nextToken(p);
} else {
if (ok != NULL)
*ok = false;
return 0;
return null_token;
}
}
static void eatDocComments(Parser* p) {
bool ok;
while (eatToken(p, TOKEN_DOC_COMMENT, &ok), ok) { }
}
static void expectSemicolon(Parser* p) {
bool ok;
eatToken(p, TOKEN_SEMICOLON, &ok);
if (ok)
return;
fprintf(stderr, "expected semicolon\n");
exit(1);
while (eatToken(p, TOKEN_DOC_COMMENT) == null_token) { }
}
static AstNodeIndex setNode(Parser* p, uint32_t i, AstNodeItem item) {
@@ -125,9 +127,7 @@ static AstNodeIndex addExtra(Parser* p, const AstNodeIndex* extra, uint32_t coun
}
static AstNodeIndex parseByteAlign(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_ALIGN, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_ALIGN) != null_token)
return null_node;
fprintf(stderr, "parseByteAlign cannot parse alginment\n");
exit(1);
@@ -135,9 +135,7 @@ static AstNodeIndex parseByteAlign(Parser* p) {
}
static AstNodeIndex parseAddrSpace(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_ADDRSPACE, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_ADDRSPACE) != null_token)
return null_node;
fprintf(stderr, "parseAddrSpace cannot parse addrspace\n");
exit(1);
@@ -145,9 +143,7 @@ static AstNodeIndex parseAddrSpace(Parser* p) {
}
static AstNodeIndex parseLinkSection(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_LINKSECTION, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_LINKSECTION) != null_token)
return null_node;
fprintf(stderr, "parseLinkSection cannot parse linksection\n");
exit(1);
@@ -155,9 +151,7 @@ static AstNodeIndex parseLinkSection(Parser* p) {
}
static AstNodeIndex parseCallconv(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_CALLCONV, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_CALLCONV) != null_token)
return null_node;
fprintf(stderr, "parseCallconv cannot parse callconv\n");
exit(1);
@@ -171,16 +165,14 @@ typedef struct {
} NodeContainerField;
static AstNodeIndex expectContainerField(Parser* p) {
eatToken(p, TOKEN_KEYWORD_COMPTIME, NULL);
eatToken(p, TOKEN_KEYWORD_COMPTIME);
const AstTokenIndex main_token = p->tok_i;
if (p->token_tags[p->tok_i] == TOKEN_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKEN_COLON)
p->tok_i += 2;
const AstNodeIndex type_expr = parseTypeExpr(p);
const AstNodeIndex align_expr = parseByteAlign(p);
bool ok;
eatToken(p, TOKEN_EQUAL, &ok);
if (ok) {
if (eatToken(p, TOKEN_EQUAL) != null_token) {
fprintf(stderr, "expectContainerField does not support expr\n");
exit(1);
}
@@ -268,7 +260,8 @@ static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
}
}
static AstNodeIndex parseSuffixOp(Parser* p) {
static AstNodeIndex parseSuffixOp(Parser* p, AstNodeIndex lhs) {
(void)lhs;
const TokenizerTag tok = p->token_tags[p->tok_i];
switch (tok) {
case TOKEN_L_BRACKET:
@@ -283,9 +276,7 @@ static AstNodeIndex parseSuffixOp(Parser* p) {
}
static AstNodeIndex parseSuffixExpr(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_ASYNC, &ok);
if (ok) {
if (eatToken(p, TOKEN_KEYWORD_ASYNC) != null_token) {
fprintf(stderr, "async not supported\n");
exit(1);
}
@@ -295,43 +286,85 @@ static AstNodeIndex parseSuffixExpr(Parser* p) {
return res;
while (true) {
const AstNodeIndex suffix_op = parseSuffixOp(p);
const AstNodeIndex suffix_op = parseSuffixOp(p, res);
if (suffix_op != 0) {
res = suffix_op;
continue;
}
eatToken(p, TOKEN_L_PAREN, &ok);
if (ok) {
fprintf(stderr, "parseSuffixExpr does not support expr with parens\n");
const AstTokenIndex lparen = eatToken(p, TOKEN_L_PAREN);
if (lparen == null_token)
return res;
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = initCleanupScratch(p);
while (true) {
if (eatToken(p, TOKEN_R_PAREN) != null_token)
break;
fprintf(stderr, "parseSuffixExpr can only parse ()\n");
exit(1);
}
// TODO more work
// const bool comma = p->token_tags[p->tok_i - 2] == TOKEN_COMMA;
return res;
const bool comma = p->token_tags[p->tok_i - 2] == TOKEN_COMMA;
const uint32_t params_len = p->scratch.len - scratch_top.old_len;
switch (params_len) {
case 0:
return addNode(
&p->nodes,
(AstNodeItem) {
.tag = comma ? AST_NODE_TAG_CALL_ONE_COMMA : AST_NODE_TAG_CALL_ONE,
.main_token = lparen,
.data = {
.lhs = res,
.rhs = 0,
},
});
case 1:
return addNode(
&p->nodes,
(AstNodeItem) {
.tag = comma ? AST_NODE_TAG_CALL_ONE_COMMA : AST_NODE_TAG_CALL_ONE,
.main_token = lparen,
.data = {
.lhs = res,
.rhs = p->scratch.arr[scratch_top.old_len],
},
});
default:;
const AstSubRange span = listToSpan(p, &p->scratch.arr[scratch_top.old_len], params_len);
return addNode(
&p->nodes,
(AstNodeItem) {
.tag = comma ? AST_NODE_TAG_CALL_COMMA : AST_NODE_TAG_CALL,
.main_token = lparen,
.data = {
.lhs = res,
.rhs = addExtra(p, (AstNodeIndex[]) { span.start, span.end }, 2),
},
});
}
}
}
static AstNodeIndex expectToken(Parser* p, TokenizerTag tag, bool* ok) {
static AstTokenIndex expectToken(Parser* p, TokenizerTag tag) {
if (p->token_tags[p->tok_i] == tag) {
if (ok != NULL)
*ok = true;
return nextToken(p);
} else {
if (ok != NULL)
*ok = false;
return 0;
fprintf(stderr, "expected token %s, got %s\n",
tokenizerGetTagString(tag),
tokenizerGetTagString(p->token_tags[p->tok_i]));
exit(1);
}
return 0; // tcc
}
static AstNodeIndex parseErrorUnionExpr(Parser* p) {
const AstNodeIndex suffix_expr = parseSuffixExpr(p);
if (suffix_expr == 0)
return null_node;
bool ok;
const AstNodeIndex bang = eatToken(p, TOKEN_BANG, &ok);
if (!ok)
const AstNodeIndex bang = eatToken(p, TOKEN_BANG);
if (bang == null_token)
return suffix_expr;
return addNode(
&p->nodes,
(AstNodeItem) {
@@ -361,18 +394,8 @@ static AstNodeIndex parseTypeExpr(Parser* p) {
static SmallSpan parseParamDeclList(Parser* p) {
// can only parse functions with no declarations
bool ok;
AstTokenIndex got_token = eatToken(p, TOKEN_L_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected (, got %s\n", tokenizerGetTagString(got_token));
exit(1);
}
got_token = eatToken(p, TOKEN_R_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected ), got %s\n", tokenizerGetTagString(got_token));
exit(1);
}
expectToken(p, TOKEN_L_PAREN);
expectToken(p, TOKEN_R_PAREN);
return (SmallSpan) {
.tag = SMALL_SPAN_ZERO_OR_ONE,
@@ -386,21 +409,20 @@ static uint32_t reserveNode(Parser* p, AstNodeTag tag) {
}
static AstNodeIndex parseFnProto(Parser* p) {
bool ok;
AstNodeIndex fn_token = eatToken(p, TOKEN_KEYWORD_FN, &ok);
if (!ok)
AstNodeIndex fn_token = eatToken(p, TOKEN_KEYWORD_FN);
if (fn_token == null_node)
return null_node;
AstNodeIndex fn_proto_index = reserveNode(p, AST_NODE_TAG_FN_PROTO);
eatToken(p, TOKEN_IDENTIFIER, NULL);
eatToken(p, TOKEN_IDENTIFIER);
SmallSpan params = parseParamDeclList(p);
const AstNodeIndex align_expr = parseByteAlign(p);
const AstNodeIndex addrspace_expr = parseAddrSpace(p);
const AstNodeIndex section_expr = parseLinkSection(p);
const AstNodeIndex callconv_expr = parseCallconv(p);
eatToken(p, TOKEN_BANG, NULL);
eatToken(p, TOKEN_BANG);
const AstNodeIndex return_type_expr = parseTypeExpr(p);
@@ -434,9 +456,8 @@ static AstTokenIndex parseBlockLabel(Parser* p) {
}
static AstNodeIndex parseForStatement(Parser* p) {
bool ok;
const AstNodeIndex for_token = eatToken(p, TOKEN_KEYWORD_FOR, &ok);
if (!ok)
const AstNodeIndex for_token = eatToken(p, TOKEN_KEYWORD_FOR);
if (for_token == null_token)
return null_node;
(void)for_token;
@@ -445,9 +466,8 @@ static AstNodeIndex parseForStatement(Parser* p) {
}
static AstNodeIndex parseWhileStatement(Parser* p) {
bool ok;
const AstNodeIndex while_token = eatToken(p, TOKEN_KEYWORD_WHILE, &ok);
if (!ok)
const AstNodeIndex while_token = eatToken(p, TOKEN_KEYWORD_WHILE);
if (while_token == null_token)
return null_node;
(void)while_token;
@@ -456,8 +476,7 @@ static AstNodeIndex parseWhileStatement(Parser* p) {
}
static AstNodeIndex parseLoopStatement(Parser* p) {
bool ok_inline_token;
eatToken(p, TOKEN_KEYWORD_INLINE, &ok_inline_token);
const AstTokenIndex inline_token = eatToken(p, TOKEN_KEYWORD_INLINE);
const AstNodeIndex for_statement = parseForStatement(p);
if (for_statement != 0)
@@ -467,10 +486,10 @@ static AstNodeIndex parseLoopStatement(Parser* p) {
if (while_statement != 0)
return while_statement;
if (!ok_inline_token)
if (inline_token == null_token)
return null_node;
fprintf(stderr, "If we've seen 'inline', there should have been a 'for' or 'while'\n");
fprintf(stderr, "seen 'inline', there should have been a 'for' or 'while'\n");
exit(1);
return 0; // tcc
}
@@ -491,8 +510,7 @@ static AstNodeIndex expectVarDeclExprStatement(Parser* p) {
static AstNodeIndex parseLabeledStatement(Parser*);
static AstNodeIndex expectStatement(Parser* p, bool allow_defer_var) {
bool ok;
if (eatToken(p, TOKEN_KEYWORD_COMPTIME, &ok), ok) {
if (eatToken(p, TOKEN_KEYWORD_COMPTIME) != null_token) {
fprintf(stderr, "expectStatement: comptime keyword not supported\n");
exit(1);
}
@@ -524,22 +542,12 @@ static AstNodeIndex expectStatement(Parser* p, bool allow_defer_var) {
}
}
typedef struct {
AstNodeIndexSlice* scratch;
uint32_t old_len;
} CleanupScratch;
static void cleanupScratch(CleanupScratch* c) { c->scratch->len = c->old_len; }
static AstNodeIndex parseBlock(Parser* p) {
bool ok;
const AstNodeIndex lbrace = eatToken(p, TOKEN_L_BRACE, &ok);
if (!ok)
const AstNodeIndex lbrace = eatToken(p, TOKEN_L_BRACE);
if (lbrace == null_token)
return null_node;
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = {
.scratch = &p->scratch,
.old_len = p->scratch.len,
};
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = initCleanupScratch(p);
while (1) {
if (p->token_tags[p->tok_i] == TOKEN_R_BRACE)
@@ -551,7 +559,7 @@ static AstNodeIndex parseBlock(Parser* p) {
break;
SLICE_APPEND(AstNodeIndex, &p->scratch, statement);
}
expectToken(p, TOKEN_R_BRACE, NULL);
expectToken(p, TOKEN_R_BRACE);
const bool semicolon = (p->token_tags[p->tok_i] - 2 == TOKEN_SEMICOLON);
const uint32_t statements_len = p->scratch.len - scratch_top.old_len;
@@ -625,13 +633,9 @@ static AstNodeIndex parseLabeledStatement(Parser* p) {
}
static AstNodeIndex parseVarDeclProto(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_CONST, &ok);
if (!ok) {
eatToken(p, TOKEN_KEYWORD_VAR, &ok);
if (!ok)
return null_node;
}
if (eatToken(p, TOKEN_KEYWORD_CONST) == null_token || eatToken(p, TOKEN_KEYWORD_VAR) == null_token)
return null_node;
fprintf(stderr, "parseVarDeclProto: parsing vars is not supported\n");
exit(1);
return 0; // tcc
@@ -652,7 +656,7 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
switch (p->token_tags[extern_export_inline_token]) {
case TOKEN_KEYWORD_EXTERN:
eatToken(p, TOKEN_STRING_LITERAL, NULL);
eatToken(p, TOKEN_STRING_LITERAL);
break;
case TOKEN_KEYWORD_EXPORT:
case TOKEN_KEYWORD_INLINE:
@@ -684,7 +688,7 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
}
}
eatToken(p, TOKEN_KEYWORD_THREADLOCAL, NULL);
eatToken(p, TOKEN_KEYWORD_THREADLOCAL);
AstNodeIndex var_decl = parseGlobalVarDecl(p);
if (var_decl != 0) {
return var_decl;
@@ -761,12 +765,9 @@ void findNextContainerMember(Parser* p) {
}
static Members parseContainerMembers(Parser* p) {
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = {
.scratch = &p->scratch,
.old_len = p->scratch.len,
};
bool ok;
while (eatToken(p, TOKEN_CONTAINER_DOC_COMMENT, &ok), ok)
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = initCleanupScratch(p);
while (eatToken(p, TOKEN_CONTAINER_DOC_COMMENT) != null_token)
;
FieldState field_state = { .tag = FIELD_STATE_NONE };