const std = @import("std"); const testing = std.testing; const Ast = std.zig.Ast; const Allocator = std.mem.Allocator; const c = @cImport({ @cInclude("ast.h"); }); const zigToken = @import("./tokenizer_test.zig").zigToken; fn zigNode(token: c_uint) Ast.Node.Tag { return switch (token) { c.AST_NODE_ROOT => .root, c.AST_NODE_USINGNAMESPACE => .@"usingnamespace", c.AST_NODE_TEST_DECL => .test_decl, c.AST_NODE_GLOBAL_VAR_DECL => .global_var_decl, c.AST_NODE_LOCAL_VAR_DECL => .local_var_decl, c.AST_NODE_SIMPLE_VAR_DECL => .simple_var_decl, c.AST_NODE_ALIGNED_VAR_DECL => .aligned_var_decl, c.AST_NODE_ERRDEFER => .@"errdefer", c.AST_NODE_DEFER => .@"defer", c.AST_NODE_CATCH => .@"catch", c.AST_NODE_FIELD_ACCESS => .field_access, c.AST_NODE_UNWRAP_OPTIONAL => .unwrap_optional, c.AST_NODE_EQUAL_EQUAL => .equal_equal, c.AST_NODE_BANG_EQUAL => .bang_equal, c.AST_NODE_LESS_THAN => .less_than, c.AST_NODE_GREATER_THAN => .greater_than, c.AST_NODE_LESS_OR_EQUAL => .less_or_equal, c.AST_NODE_GREATER_OR_EQUAL => .greater_or_equal, c.AST_NODE_ASSIGN_MUL => .assign_mul, c.AST_NODE_ASSIGN_DIV => .assign_div, c.AST_NODE_ASSIGN_MOD => .assign_mod, c.AST_NODE_ASSIGN_ADD => .assign_add, c.AST_NODE_ASSIGN_SUB => .assign_sub, c.AST_NODE_ASSIGN_SHL => .assign_shl, c.AST_NODE_ASSIGN_SHL_SAT => .assign_shl_sat, c.AST_NODE_ASSIGN_SHR => .assign_shr, c.AST_NODE_ASSIGN_BIT_AND => .assign_bit_and, c.AST_NODE_ASSIGN_BIT_XOR => .assign_bit_xor, c.AST_NODE_ASSIGN_BIT_OR => .assign_bit_or, c.AST_NODE_ASSIGN_MUL_WRAP => .assign_mul_wrap, c.AST_NODE_ASSIGN_ADD_WRAP => .assign_add_wrap, c.AST_NODE_ASSIGN_SUB_WRAP => .assign_sub_wrap, c.AST_NODE_ASSIGN_MUL_SAT => .assign_mul_sat, c.AST_NODE_ASSIGN_ADD_SAT => .assign_add_sat, c.AST_NODE_ASSIGN_SUB_SAT => .assign_sub_sat, c.AST_NODE_ASSIGN => .assign, c.AST_NODE_ASSIGN_DESTRUCTURE => .assign_destructure, c.AST_NODE_MERGE_ERROR_SETS => .merge_error_sets, c.AST_NODE_MUL => .mul, c.AST_NODE_DIV => .div, c.AST_NODE_MOD => .mod, c.AST_NODE_ARRAY_MULT => .array_mult, c.AST_NODE_MUL_WRAP => .mul_wrap, c.AST_NODE_MUL_SAT => .mul_sat, c.AST_NODE_ADD => .add, c.AST_NODE_SUB => .sub, c.AST_NODE_ARRAY_CAT => .array_cat, c.AST_NODE_ADD_WRAP => .add_wrap, c.AST_NODE_SUB_WRAP => .sub_wrap, c.AST_NODE_ADD_SAT => .add_sat, c.AST_NODE_SUB_SAT => .sub_sat, c.AST_NODE_SHL => .shl, c.AST_NODE_SHL_SAT => .shl_sat, c.AST_NODE_SHR => .shr, c.AST_NODE_BIT_AND => .bit_and, c.AST_NODE_BIT_XOR => .bit_xor, c.AST_NODE_BIT_OR => .bit_or, c.AST_NODE_ORELSE => .@"orelse", c.AST_NODE_BOOL_AND => .bool_and, c.AST_NODE_BOOL_OR => .bool_or, c.AST_NODE_BOOL_NOT => .bool_not, c.AST_NODE_NEGATION => .negation, c.AST_NODE_BIT_NOT => .bit_not, c.AST_NODE_NEGATION_WRAP => .negation_wrap, c.AST_NODE_ADDRESS_OF => .address_of, c.AST_NODE_TRY => .@"try", c.AST_NODE_AWAIT => .@"await", c.AST_NODE_OPTIONAL_TYPE => .optional_type, c.AST_NODE_ARRAY_TYPE => .array_type, c.AST_NODE_ARRAY_TYPE_SENTINEL => .array_type_sentinel, c.AST_NODE_PTR_TYPE_ALIGNED => .ptr_type_aligned, c.AST_NODE_PTR_TYPE_SENTINEL => .ptr_type_sentinel, c.AST_NODE_PTR_TYPE => .ptr_type, c.AST_NODE_PTR_TYPE_BIT_RANGE => .ptr_type_bit_range, c.AST_NODE_SLICE_OPEN => .slice_open, c.AST_NODE_SLICE => .slice, c.AST_NODE_SLICE_SENTINEL => .slice_sentinel, c.AST_NODE_DEREF => .deref, c.AST_NODE_ARRAY_ACCESS => .array_access, c.AST_NODE_ARRAY_INIT_ONE => .array_init_one, c.AST_NODE_ARRAY_INIT_ONE_COMMA => .array_init_one_comma, c.AST_NODE_ARRAY_INIT_DOT_TWO => .array_init_dot_two, c.AST_NODE_ARRAY_INIT_DOT_TWO_COMMA => .array_init_dot_two_comma, c.AST_NODE_ARRAY_INIT_DOT => .array_init_dot, c.AST_NODE_ARRAY_INIT_DOT_COMMA => .array_init_dot_comma, c.AST_NODE_ARRAY_INIT => .array_init, c.AST_NODE_ARRAY_INIT_COMMA => .array_init_comma, c.AST_NODE_STRUCT_INIT_ONE => .struct_init_one, c.AST_NODE_STRUCT_INIT_ONE_COMMA => .struct_init_one_comma, c.AST_NODE_STRUCT_INIT_DOT_TWO => .struct_init_dot_two, c.AST_NODE_STRUCT_INIT_DOT_TWO_COMMA => .struct_init_dot_two_comma, c.AST_NODE_STRUCT_INIT_DOT => .struct_init_dot, c.AST_NODE_STRUCT_INIT_DOT_COMMA => .struct_init_dot_comma, c.AST_NODE_STRUCT_INIT => .struct_init, c.AST_NODE_STRUCT_INIT_COMMA => .struct_init_comma, c.AST_NODE_CALL_ONE => .call_one, c.AST_NODE_CALL_ONE_COMMA => .call_one_comma, c.AST_NODE_ASYNC_CALL_ONE => .async_call_one, c.AST_NODE_ASYNC_CALL_ONE_COMMA => .async_call_one_comma, c.AST_NODE_CALL => .call, c.AST_NODE_CALL_COMMA => .call_comma, c.AST_NODE_ASYNC_CALL => .async_call, c.AST_NODE_ASYNC_CALL_COMMA => .async_call_comma, c.AST_NODE_SWITCH => .@"switch", c.AST_NODE_SWITCH_COMMA => .switch_comma, c.AST_NODE_SWITCH_CASE_ONE => .switch_case_one, c.AST_NODE_SWITCH_CASE_INLINE_ONE => .switch_case_inline_one, c.AST_NODE_SWITCH_CASE => .switch_case, c.AST_NODE_SWITCH_CASE_INLINE => .switch_case_inline, c.AST_NODE_SWITCH_RANGE => .switch_range, c.AST_NODE_WHILE_SIMPLE => .while_simple, c.AST_NODE_WHILE_CONT => .while_cont, c.AST_NODE_WHILE => .@"while", c.AST_NODE_FOR_SIMPLE => .for_simple, c.AST_NODE_FOR => .@"for", c.AST_NODE_FOR_RANGE => .for_range, c.AST_NODE_IF_SIMPLE => .if_simple, c.AST_NODE_IF => .@"if", c.AST_NODE_SUSPEND => .@"suspend", c.AST_NODE_RESUME => .@"resume", c.AST_NODE_CONTINUE => .@"continue", c.AST_NODE_BREAK => .@"break", c.AST_NODE_RETURN => .@"return", c.AST_NODE_FN_PROTO_SIMPLE => .fn_proto_simple, c.AST_NODE_FN_PROTO_MULTI => .fn_proto_multi, c.AST_NODE_FN_PROTO_ONE => .fn_proto_one, c.AST_NODE_FN_PROTO => .fn_proto, c.AST_NODE_FN_DECL => .fn_decl, c.AST_NODE_ANYFRAME_TYPE => .anyframe_type, c.AST_NODE_ANYFRAME_LITERAL => .anyframe_literal, c.AST_NODE_CHAR_LITERAL => .char_literal, c.AST_NODE_NUMBER_LITERAL => .number_literal, c.AST_NODE_UNREACHABLE_LITERAL => .unreachable_literal, c.AST_NODE_IDENTIFIER => .identifier, c.AST_NODE_ENUM_LITERAL => .enum_literal, c.AST_NODE_STRING_LITERAL => .string_literal, c.AST_NODE_MULTILINE_STRING_LITERAL => .multiline_string_literal, c.AST_NODE_GROUPED_EXPRESSION => .grouped_expression, c.AST_NODE_BUILTIN_CALL_TWO => .builtin_call_two, c.AST_NODE_BUILTIN_CALL_TWO_COMMA => .builtin_call_two_comma, c.AST_NODE_BUILTIN_CALL => .builtin_call, c.AST_NODE_BUILTIN_CALL_COMMA => .builtin_call_comma, c.AST_NODE_ERROR_SET_DECL => .error_set_decl, c.AST_NODE_CONTAINER_DECL => .container_decl, c.AST_NODE_CONTAINER_DECL_TRAILING => .container_decl_trailing, c.AST_NODE_CONTAINER_DECL_TWO => .container_decl_two, c.AST_NODE_CONTAINER_DECL_TWO_TRAILING => .container_decl_two_trailing, c.AST_NODE_CONTAINER_DECL_ARG => .container_decl_arg, c.AST_NODE_CONTAINER_DECL_ARG_TRAILING => .container_decl_arg_trailing, c.AST_NODE_TAGGED_UNION => .tagged_union, c.AST_NODE_TAGGED_UNION_TRAILING => .tagged_union_trailing, c.AST_NODE_TAGGED_UNION_TWO => .tagged_union_two, c.AST_NODE_TAGGED_UNION_TWO_TRAILING => .tagged_union_two_trailing, c.AST_NODE_TAGGED_UNION_ENUM_TAG => .tagged_union_enum_tag, c.AST_NODE_TAGGED_UNION_ENUM_TAG_TRAILING => .tagged_union_enum_tag_trailing, c.AST_NODE_CONTAINER_FIELD_INIT => .container_field_init, c.AST_NODE_CONTAINER_FIELD_ALIGN => .container_field_align, c.AST_NODE_CONTAINER_FIELD => .container_field, c.AST_NODE_COMPTIME => .@"comptime", c.AST_NODE_NOSUSPEND => .@"nosuspend", c.AST_NODE_BLOCK_TWO => .block_two, c.AST_NODE_BLOCK_TWO_SEMICOLON => .block_two_semicolon, c.AST_NODE_BLOCK => .block, c.AST_NODE_BLOCK_SEMICOLON => .block_semicolon, c.AST_NODE_ASM_SIMPLE => .asm_simple, c.AST_NODE_ASM => .@"asm", c.AST_NODE_ASM_OUTPUT => .asm_output, c.AST_NODE_ASM_INPUT => .asm_input, c.AST_NODE_ERROR_VALUE => .error_value, c.AST_NODE_ERROR_UNION => .error_union, else => undefined, }; } // zigAst converts a c.Ast to std.Zig.Ast. The resulting Ast should be freed with deinit(). fn zigAst(gpa: Allocator, c_ast: c.Ast) !Ast { var tokens = Ast.TokenList{}; try tokens.ensureTotalCapacity(gpa, c_ast.tokens.len); errdefer tokens.deinit(gpa); for (0..c_ast.tokens.len) |i| tokens.set(i, .{ .tag = zigToken(c_ast.tokens.tags[i]), .start = c_ast.tokens.starts[i], }); var nodes = Ast.NodeList{}; try nodes.ensureTotalCapacity(gpa, c_ast.nodes.len); errdefer nodes.deinit(gpa); for (0..c_ast.nodes.len) |i| nodes.set(i, .{ .tag = zigNode(c_ast.nodes.tags[i]), .main_token = c_ast.nodes.main_tokens[i], .data = Ast.Node.Data{ .lhs = c_ast.nodes.datas[i].lhs, .rhs = c_ast.nodes.datas[i].rhs, }, }); var extra_data = try gpa.alloc(Ast.Node.Index, c_ast.extra_data.len); errdefer gpa.free(extra_data); std.mem.copyForwards( Ast.Node.Index, extra_data[0..], c_ast.extra_data.arr[0..c_ast.extra_data.len], ); // creating a dummy `errors` slice, so deinit can free it. const errors = try gpa.alloc(Ast.Error, 0); errdefer gpa.deinit(errors); return Ast{ .source = c_ast.source[0..c_ast.source_len :0], .mode = .zig, .tokens = tokens.slice(), .nodes = nodes.slice(), .extra_data = extra_data, .errors = errors, }; } test "Ast header smoke test" { try std.testing.expectEqual(zigNode(c.AST_NODE_IF), Ast.Node.Tag.@"if"); } // copy-past from parser_test.zig const mem = std.mem; const print = std.debug.print; const io = std.io; const maxInt = std.math.maxInt; var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: [:0]const u8, allocator: mem.Allocator, anything_changed: *bool) ![]u8 { const stderr = io.getStdErr().writer(); //var tree = try std.zig.Ast.parse(allocator, source, .zig); const c_tree = c.astParse(source, @intCast(source.len)); var tree = try zigAst(allocator, c_tree); defer tree.deinit(allocator); for (tree.errors) |parse_error| { const loc = tree.tokenLocation(0, parse_error.token); try stderr.print("(memory buffer):{d}:{d}: error: ", .{ loc.line + 1, loc.column + 1 }); try tree.renderError(parse_error, stderr); try stderr.print("\n{s}\n", .{source[loc.line_start..loc.line_end]}); { var i: usize = 0; while (i < loc.column) : (i += 1) { try stderr.writeAll(" "); } try stderr.writeAll("^"); } try stderr.writeAll("\n"); } if (tree.errors.len != 0) { return error.ParseError; } const formatted = try tree.render(allocator); anything_changed.* = !mem.eql(u8, formatted, source); return formatted; } fn testTransformImpl(allocator: mem.Allocator, fba: *std.heap.FixedBufferAllocator, source: [:0]const u8, expected_source: []const u8) !void { // reset the fixed buffer allocator each run so that it can be re-used for each // iteration of the failing index fba.reset(); var anything_changed: bool = undefined; const result_source = try testParse(source, allocator, &anything_changed); try std.testing.expectEqualStrings(expected_source, result_source); const changes_expected = source.ptr != expected_source.ptr; if (anything_changed != changes_expected) { print("std.zig.render returned {} instead of {}\n", .{ anything_changed, changes_expected }); return error.TestFailed; } try std.testing.expect(anything_changed == changes_expected); allocator.free(result_source); } fn testTransform(source: [:0]const u8, expected_source: []const u8) !void { var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); return std.testing.checkAllAllocationFailures(fixed_allocator.allocator(), testTransformImpl, .{ &fixed_allocator, source, expected_source }); } fn testCanonical(source: [:0]const u8) !void { return testTransform(source, source); } test "zig fmt: remove extra whitespace at start and end of file with comment between" { if (true) return error.SkipZigTest; try testTransform( \\ \\ \\// hello \\ \\ , \\// hello \\ ); }