motiejus/zig0
1
Fork 0
Code Releases Activity

more parser

Browse Source
This commit is contained in:
Motiejus Jakštys
2024-12-31 19:14:12 +02:00
parent 3264d1747e
commit 85dfbe9d09
4 changed files with 136 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
build.zig
View File

@@ -77,14 +77,15 @@ pub fn build(b: *std.Build) !void {
test_exe.addIncludePath(b.path("."));
test_step.dependOn(&b.addRunArtifact(test_exe).step);
const lint_step = b.step("lint", "Run linters");
const clang_format = b.addSystemCommand(&.{"clang-format"});
clang_format.addArgs(&.{ "-Werror", "-i" });
const fmt_step = b.step("fmt", "clang-format");
const clang_format = b.addSystemCommand(&.{ "clang-format", "-Werror", "-i" });
for (all_c_files ++ headers) |f| clang_format.addFileArg(b.path(f));
lint_step.dependOn(&clang_format.step);
fmt_step.dependOn(&clang_format.step);
const clang_analyze = b.addSystemCommand(&.{"clang"});
clang_analyze.addArgs(&.{
const lint_step = b.step("lint", "Run linters");
const clang_analyze = b.addSystemCommand(&.{
"clang",
"--analyze",
"--analyzer-output",
"text",
@@ -94,13 +95,19 @@ pub fn build(b: *std.Build) !void {
for (all_c_files) |cfile| clang_analyze.addFileArg(b.path(cfile));
lint_step.dependOn(&clang_analyze.step);
const gcc_analyze = b.addSystemCommand(&.{"gcc"});
gcc_analyze.addArgs(&.{ "--analyzer", "-Werror", "-o", "/dev/null" });
const gcc_analyze = b.addSystemCommand(&.{
"gcc",
"--analyzer",
"-Wno-analyzer-malloc-leak", // TODO remove when wiring is complete and everything's free()d
"-Werror",
"-o",
"/dev/null",
});
for (all_c_files) |cfile| gcc_analyze.addFileArg(b.path(cfile));
lint_step.dependOn(&gcc_analyze.step);
const cppcheck = b.addSystemCommand(&.{"cppcheck"});
cppcheck.addArgs(&.{
const cppcheck = b.addSystemCommand(&.{
"cppcheck",
"--quiet",
"--error-exitcode=1",
"--check-level=exhaustive",

6
common.h
View File

@@ -27,10 +27,12 @@
}
#define SLICE_RESIZE(Type, slice, new_cap) ({ \
uint32_t cap = (new_cap); \
const uint32_t cap = (new_cap); \
Type* new_arr = realloc((slice)->arr, cap * sizeof(Type)); \
if (!new_arr) \
if (new_arr == NULL) { \
free((slice)->arr); \
exit(1); \
} \
(slice)->arr = new_arr; \
(slice)->cap = cap; \
})

227
parser.c
View File

@@ -7,12 +7,13 @@
#include "parser.h"
const AstNodeIndex null_node = 0;
const AstTokenIndex null_token = ~(AstTokenIndex)(0);
typedef struct {
enum {
FIELD_STATE_NONE,
FIELD_STATE_SEEN,
FIELD_STATE_END // sets "end"
FIELD_STATE_END
} tag;
union {
uint32_t end;
@@ -30,6 +31,22 @@ typedef struct {
} payload;
} SmallSpan;
typedef struct {
AstNodeIndexSlice* scratch;
uint32_t old_len;
} CleanupScratch;
static CleanupScratch initCleanupScratch(Parser* p) {
return (CleanupScratch) {
.scratch = &p->scratch,
.old_len = p->scratch.len,
};
}
static void cleanupScratch(CleanupScratch* c) {
c->scratch->len = c->old_len;
}
static AstSubRange listToSpan(Parser* p, const AstNodeIndex* list, uint32_t count) {
SLICE_ENSURE_CAPACITY(AstNodeIndex, &p->extra_data, count);
memcpy(&p->extra_data.arr, list, count * sizeof(AstNodeIndex));
@@ -60,31 +77,16 @@ static AstTokenIndex nextToken(Parser* p) {
return p->tok_i++;
}
static AstTokenIndex eatToken(Parser* p, TokenizerTag tag, bool* ok) {
static AstTokenIndex eatToken(Parser* p, TokenizerTag tag) {
if (p->token_tags[p->tok_i] == tag) {
if (ok != NULL)
*ok = true;
return nextToken(p);
} else {
if (ok != NULL)
*ok = false;
return 0;
return null_token;
}
}
static void eatDocComments(Parser* p) {
bool ok;
while (eatToken(p, TOKEN_DOC_COMMENT, &ok), ok) { }
}
static void expectSemicolon(Parser* p) {
bool ok;
eatToken(p, TOKEN_SEMICOLON, &ok);
if (ok)
return;
fprintf(stderr, "expected semicolon\n");
exit(1);
while (eatToken(p, TOKEN_DOC_COMMENT) == null_token) { }
}
static AstNodeIndex setNode(Parser* p, uint32_t i, AstNodeItem item) {
@@ -125,9 +127,7 @@ static AstNodeIndex addExtra(Parser* p, const AstNodeIndex* extra, uint32_t coun
}
static AstNodeIndex parseByteAlign(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_ALIGN, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_ALIGN) != null_token)
return null_node;
fprintf(stderr, "parseByteAlign cannot parse alginment\n");
exit(1);
@@ -135,9 +135,7 @@ static AstNodeIndex parseByteAlign(Parser* p) {
}
static AstNodeIndex parseAddrSpace(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_ADDRSPACE, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_ADDRSPACE) != null_token)
return null_node;
fprintf(stderr, "parseAddrSpace cannot parse addrspace\n");
exit(1);
@@ -145,9 +143,7 @@ static AstNodeIndex parseAddrSpace(Parser* p) {
}
static AstNodeIndex parseLinkSection(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_LINKSECTION, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_LINKSECTION) != null_token)
return null_node;
fprintf(stderr, "parseLinkSection cannot parse linksection\n");
exit(1);
@@ -155,9 +151,7 @@ static AstNodeIndex parseLinkSection(Parser* p) {
}
static AstNodeIndex parseCallconv(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_CALLCONV, &ok);
if (!ok)
if (eatToken(p, TOKEN_KEYWORD_CALLCONV) != null_token)
return null_node;
fprintf(stderr, "parseCallconv cannot parse callconv\n");
exit(1);
@@ -171,16 +165,14 @@ typedef struct {
} NodeContainerField;
static AstNodeIndex expectContainerField(Parser* p) {
eatToken(p, TOKEN_KEYWORD_COMPTIME, NULL);
eatToken(p, TOKEN_KEYWORD_COMPTIME);
const AstTokenIndex main_token = p->tok_i;
if (p->token_tags[p->tok_i] == TOKEN_IDENTIFIER && p->token_tags[p->tok_i + 1] == TOKEN_COLON)
p->tok_i += 2;
const AstNodeIndex type_expr = parseTypeExpr(p);
const AstNodeIndex align_expr = parseByteAlign(p);
bool ok;
eatToken(p, TOKEN_EQUAL, &ok);
if (ok) {
if (eatToken(p, TOKEN_EQUAL) != null_token) {
fprintf(stderr, "expectContainerField does not support expr\n");
exit(1);
}
@@ -268,7 +260,8 @@ static AstNodeIndex parsePrimaryTypeExpr(Parser* p) {
}
}
static AstNodeIndex parseSuffixOp(Parser* p) {
static AstNodeIndex parseSuffixOp(Parser* p, AstNodeIndex lhs) {
(void)lhs;
const TokenizerTag tok = p->token_tags[p->tok_i];
switch (tok) {
case TOKEN_L_BRACKET:
@@ -283,9 +276,7 @@ static AstNodeIndex parseSuffixOp(Parser* p) {
}
static AstNodeIndex parseSuffixExpr(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_ASYNC, &ok);
if (ok) {
if (eatToken(p, TOKEN_KEYWORD_ASYNC) != null_token) {
fprintf(stderr, "async not supported\n");
exit(1);
}
@@ -295,43 +286,85 @@ static AstNodeIndex parseSuffixExpr(Parser* p) {
return res;
while (true) {
const AstNodeIndex suffix_op = parseSuffixOp(p);
const AstNodeIndex suffix_op = parseSuffixOp(p, res);
if (suffix_op != 0) {
res = suffix_op;
continue;
}
eatToken(p, TOKEN_L_PAREN, &ok);
if (ok) {
fprintf(stderr, "parseSuffixExpr does not support expr with parens\n");
const AstTokenIndex lparen = eatToken(p, TOKEN_L_PAREN);
if (lparen == null_token)
return res;
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = initCleanupScratch(p);
while (true) {
if (eatToken(p, TOKEN_R_PAREN) != null_token)
break;
fprintf(stderr, "parseSuffixExpr can only parse ()\n");
exit(1);
}
// TODO more work
// const bool comma = p->token_tags[p->tok_i - 2] == TOKEN_COMMA;
return res;
const bool comma = p->token_tags[p->tok_i - 2] == TOKEN_COMMA;
const uint32_t params_len = p->scratch.len - scratch_top.old_len;
switch (params_len) {
case 0:
return addNode(
&p->nodes,
(AstNodeItem) {
.tag = comma ? AST_NODE_TAG_CALL_ONE_COMMA : AST_NODE_TAG_CALL_ONE,
.main_token = lparen,
.data = {
.lhs = res,
.rhs = 0,
},
});
case 1:
return addNode(
&p->nodes,
(AstNodeItem) {
.tag = comma ? AST_NODE_TAG_CALL_ONE_COMMA : AST_NODE_TAG_CALL_ONE,
.main_token = lparen,
.data = {
.lhs = res,
.rhs = p->scratch.arr[scratch_top.old_len],
},
});
default:;
const AstSubRange span = listToSpan(p, &p->scratch.arr[scratch_top.old_len], params_len);
return addNode(
&p->nodes,
(AstNodeItem) {
.tag = comma ? AST_NODE_TAG_CALL_COMMA : AST_NODE_TAG_CALL,
.main_token = lparen,
.data = {
.lhs = res,
.rhs = addExtra(p, (AstNodeIndex[]) { span.start, span.end }, 2),
},
});
}
}
}
static AstNodeIndex expectToken(Parser* p, TokenizerTag tag, bool* ok) {
static AstTokenIndex expectToken(Parser* p, TokenizerTag tag) {
if (p->token_tags[p->tok_i] == tag) {
if (ok != NULL)
*ok = true;
return nextToken(p);
} else {
if (ok != NULL)
*ok = false;
return 0;
fprintf(stderr, "expected token %s, got %s\n",
tokenizerGetTagString(tag),
tokenizerGetTagString(p->token_tags[p->tok_i]));
exit(1);
}
return 0; // tcc
}
static AstNodeIndex parseErrorUnionExpr(Parser* p) {
const AstNodeIndex suffix_expr = parseSuffixExpr(p);
if (suffix_expr == 0)
return null_node;
bool ok;
const AstNodeIndex bang = eatToken(p, TOKEN_BANG, &ok);
if (!ok)
const AstNodeIndex bang = eatToken(p, TOKEN_BANG);
if (bang == null_token)
return suffix_expr;
return addNode(
&p->nodes,
(AstNodeItem) {
@@ -361,18 +394,8 @@ static AstNodeIndex parseTypeExpr(Parser* p) {
static SmallSpan parseParamDeclList(Parser* p) {
// can only parse functions with no declarations
bool ok;
AstTokenIndex got_token = eatToken(p, TOKEN_L_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected (, got %s\n", tokenizerGetTagString(got_token));
exit(1);
}
got_token = eatToken(p, TOKEN_R_PAREN, &ok);
if (!ok) {
fprintf(stderr, "expected ), got %s\n", tokenizerGetTagString(got_token));
exit(1);
}
expectToken(p, TOKEN_L_PAREN);
expectToken(p, TOKEN_R_PAREN);
return (SmallSpan) {
.tag = SMALL_SPAN_ZERO_OR_ONE,
@@ -386,21 +409,20 @@ static uint32_t reserveNode(Parser* p, AstNodeTag tag) {
}
static AstNodeIndex parseFnProto(Parser* p) {
bool ok;
AstNodeIndex fn_token = eatToken(p, TOKEN_KEYWORD_FN, &ok);
if (!ok)
AstNodeIndex fn_token = eatToken(p, TOKEN_KEYWORD_FN);
if (fn_token == null_node)
return null_node;
AstNodeIndex fn_proto_index = reserveNode(p, AST_NODE_TAG_FN_PROTO);
eatToken(p, TOKEN_IDENTIFIER, NULL);
eatToken(p, TOKEN_IDENTIFIER);
SmallSpan params = parseParamDeclList(p);
const AstNodeIndex align_expr = parseByteAlign(p);
const AstNodeIndex addrspace_expr = parseAddrSpace(p);
const AstNodeIndex section_expr = parseLinkSection(p);
const AstNodeIndex callconv_expr = parseCallconv(p);
eatToken(p, TOKEN_BANG, NULL);
eatToken(p, TOKEN_BANG);
const AstNodeIndex return_type_expr = parseTypeExpr(p);
@@ -434,9 +456,8 @@ static AstTokenIndex parseBlockLabel(Parser* p) {
}
static AstNodeIndex parseForStatement(Parser* p) {
bool ok;
const AstNodeIndex for_token = eatToken(p, TOKEN_KEYWORD_FOR, &ok);
if (!ok)
const AstNodeIndex for_token = eatToken(p, TOKEN_KEYWORD_FOR);
if (for_token == null_token)
return null_node;
(void)for_token;
@@ -445,9 +466,8 @@ static AstNodeIndex parseForStatement(Parser* p) {
}
static AstNodeIndex parseWhileStatement(Parser* p) {
bool ok;
const AstNodeIndex while_token = eatToken(p, TOKEN_KEYWORD_WHILE, &ok);
if (!ok)
const AstNodeIndex while_token = eatToken(p, TOKEN_KEYWORD_WHILE);
if (while_token == null_token)
return null_node;
(void)while_token;
@@ -456,8 +476,7 @@ static AstNodeIndex parseWhileStatement(Parser* p) {
}
static AstNodeIndex parseLoopStatement(Parser* p) {
bool ok_inline_token;
eatToken(p, TOKEN_KEYWORD_INLINE, &ok_inline_token);
const AstTokenIndex inline_token = eatToken(p, TOKEN_KEYWORD_INLINE);
const AstNodeIndex for_statement = parseForStatement(p);
if (for_statement != 0)
@@ -467,10 +486,10 @@ static AstNodeIndex parseLoopStatement(Parser* p) {
if (while_statement != 0)
return while_statement;
if (!ok_inline_token)
if (inline_token == null_token)
return null_node;
fprintf(stderr, "If we've seen 'inline', there should have been a 'for' or 'while'\n");
fprintf(stderr, "seen 'inline', there should have been a 'for' or 'while'\n");
exit(1);
return 0; // tcc
}
@@ -491,8 +510,7 @@ static AstNodeIndex expectVarDeclExprStatement(Parser* p) {
static AstNodeIndex parseLabeledStatement(Parser*);
static AstNodeIndex expectStatement(Parser* p, bool allow_defer_var) {
bool ok;
if (eatToken(p, TOKEN_KEYWORD_COMPTIME, &ok), ok) {
if (eatToken(p, TOKEN_KEYWORD_COMPTIME) != null_token) {
fprintf(stderr, "expectStatement: comptime keyword not supported\n");
exit(1);
}
@@ -524,22 +542,12 @@ static AstNodeIndex expectStatement(Parser* p, bool allow_defer_var) {
}
}
typedef struct {
AstNodeIndexSlice* scratch;
uint32_t old_len;
} CleanupScratch;
static void cleanupScratch(CleanupScratch* c) { c->scratch->len = c->old_len; }
static AstNodeIndex parseBlock(Parser* p) {
bool ok;
const AstNodeIndex lbrace = eatToken(p, TOKEN_L_BRACE, &ok);
if (!ok)
const AstNodeIndex lbrace = eatToken(p, TOKEN_L_BRACE);
if (lbrace == null_token)
return null_node;
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = {
.scratch = &p->scratch,
.old_len = p->scratch.len,
};
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = initCleanupScratch(p);
while (1) {
if (p->token_tags[p->tok_i] == TOKEN_R_BRACE)
@@ -551,7 +559,7 @@ static AstNodeIndex parseBlock(Parser* p) {
break;
SLICE_APPEND(AstNodeIndex, &p->scratch, statement);
}
expectToken(p, TOKEN_R_BRACE, NULL);
expectToken(p, TOKEN_R_BRACE);
const bool semicolon = (p->token_tags[p->tok_i] - 2 == TOKEN_SEMICOLON);
const uint32_t statements_len = p->scratch.len - scratch_top.old_len;
@@ -625,13 +633,9 @@ static AstNodeIndex parseLabeledStatement(Parser* p) {
}
static AstNodeIndex parseVarDeclProto(Parser* p) {
bool ok;
eatToken(p, TOKEN_KEYWORD_CONST, &ok);
if (!ok) {
eatToken(p, TOKEN_KEYWORD_VAR, &ok);
if (!ok)
return null_node;
}
if (eatToken(p, TOKEN_KEYWORD_CONST) == null_token || eatToken(p, TOKEN_KEYWORD_VAR) == null_token)
return null_node;
fprintf(stderr, "parseVarDeclProto: parsing vars is not supported\n");
exit(1);
return 0; // tcc
@@ -652,7 +656,7 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
switch (p->token_tags[extern_export_inline_token]) {
case TOKEN_KEYWORD_EXTERN:
eatToken(p, TOKEN_STRING_LITERAL, NULL);
eatToken(p, TOKEN_STRING_LITERAL);
break;
case TOKEN_KEYWORD_EXPORT:
case TOKEN_KEYWORD_INLINE:
@@ -684,7 +688,7 @@ static AstNodeIndex expectTopLevelDecl(Parser* p) {
}
}
eatToken(p, TOKEN_KEYWORD_THREADLOCAL, NULL);
eatToken(p, TOKEN_KEYWORD_THREADLOCAL);
AstNodeIndex var_decl = parseGlobalVarDecl(p);
if (var_decl != 0) {
return var_decl;
@@ -761,12 +765,9 @@ void findNextContainerMember(Parser* p) {
}
static Members parseContainerMembers(Parser* p) {
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = {
.scratch = &p->scratch,
.old_len = p->scratch.len,
};
bool ok;
while (eatToken(p, TOKEN_CONTAINER_DOC_COMMENT, &ok), ok)
CleanupScratch scratch_top __attribute__((__cleanup__(cleanupScratch))) = initCleanupScratch(p);
while (eatToken(p, TOKEN_CONTAINER_DOC_COMMENT) != null_token)
;
FieldState field_state = { .tag = FIELD_STATE_NONE };

2
tokenizer.h
View File

@@ -4,7 +4,7 @@
#include <stdbool.h>
#include <stdint.h>
#define TOKENIZER_FOREACH_TAG_ENUM(TAG) \
#define TOKENIZER_FOREACH_TAG_ENUM(TAG) \
TAG(TOKEN_INVALID) \
TAG(TOKEN_INVALID_PERIODASTERISKS) \
TAG(TOKEN_IDENTIFIER) \