blob a1dd659e (23141B) - Raw
1 const std = @import("std"); 2 const link = @import("link.zig"); 3 const Module = @import("Module.zig"); 4 const Allocator = std.mem.Allocator; 5 const zir = @import("zir.zig"); 6 const Package = @import("Package.zig"); 7 8 test "self-hosted" { 9 var ctx = TestContext.init(); 10 defer ctx.deinit(); 11 12 try @import("stage2_tests").addCases(&ctx); 13 14 try ctx.run(); 15 } 16 17 const ErrorMsg = struct { 18 msg: []const u8, 19 line: u32, 20 column: u32, 21 }; 22 23 pub const TestContext = struct { 24 /// TODO: find a way to treat cases as individual tests (shouldn't show "1 test passed" if there are 200 cases) 25 cases: std.ArrayList(Case), 26 27 pub const Update = struct { 28 /// The input to the current update. We simulate an incremental update 29 /// with the file's contents changed to this value each update. 30 /// 31 /// This value can change entirely between updates, which would be akin 32 /// to deleting the source file and creating a new one from scratch; or 33 /// you can keep it mostly consistent, with small changes, testing the 34 /// effects of the incremental compilation. 35 src: [:0]const u8, 36 case: union(enum) { 37 /// A transformation update transforms the input and tests against 38 /// the expected output ZIR. 39 Transformation: [:0]const u8, 40 /// An error update attempts to compile bad code, and ensures that it 41 /// fails to compile, and for the expected reasons. 42 /// A slice containing the expected errors *in sequential order*. 43 Error: []const ErrorMsg, 44 /// An execution update compiles and runs the input, testing the 45 /// stdout against the expected results 46 /// This is a slice containing the expected message. 47 Execution: []const u8, 48 }, 49 }; 50 51 pub const TestType = enum { 52 Zig, 53 ZIR, 54 }; 55 56 /// A Case consists of a set of *updates*. The same Module is used for each 57 /// update, so each update's source is treated as a single file being 58 /// updated by the test harness and incrementally compiled. 59 pub const Case = struct { 60 /// The name of the test case. This is shown if a test fails, and 61 /// otherwise ignored. 62 name: []const u8, 63 /// The platform the test targets. For non-native platforms, an emulator 64 /// such as QEMU is required for tests to complete. 65 target: std.zig.CrossTarget, 66 /// In order to be able to run e.g. Execution updates, this must be set 67 /// to Executable. This is ignored when generating C output. 68 output_mode: std.builtin.OutputMode, 69 updates: std.ArrayList(Update), 70 extension: TestType, 71 c_standard: ?Module.CStandard = null, 72 73 /// Adds a subcase in which the module is updated with `src`, and the 74 /// resulting ZIR is validated against `result`. 75 pub fn addTransform(self: *Case, src: [:0]const u8, result: [:0]const u8) void { 76 self.updates.append(.{ 77 .src = src, 78 .case = .{ .Transformation = result }, 79 }) catch unreachable; 80 } 81 82 /// Adds a subcase in which the module is updated with `src`, compiled, 83 /// run, and the output is tested against `result`. 84 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void { 85 self.updates.append(.{ 86 .src = src, 87 .case = .{ .Execution = result }, 88 }) catch unreachable; 89 } 90 91 /// Adds a subcase in which the module is updated with `src`, which 92 /// should contain invalid input, and ensures that compilation fails 93 /// for the expected reasons, given in sequential order in `errors` in 94 /// the form `:line:column: error: message`. 95 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void { 96 var array = self.updates.allocator.alloc(ErrorMsg, errors.len) catch unreachable; 97 for (errors) |e, i| { 98 if (e[0] != ':') { 99 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); 100 } 101 var cur = e[1..]; 102 var line_index = std.mem.indexOf(u8, cur, ":"); 103 if (line_index == null) { 104 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); 105 } 106 const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number"); 107 cur = cur[line_index.? + 1 ..]; 108 const column_index = std.mem.indexOf(u8, cur, ":"); 109 if (column_index == null) { 110 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); 111 } 112 const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number"); 113 cur = cur[column_index.? + 2 ..]; 114 if (!std.mem.eql(u8, cur[0..7], "error: ")) { 115 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n"); 116 } 117 const msg = cur[7..]; 118 119 if (line == 0 or column == 0) { 120 @panic("Invalid test: error line and column must be specified starting at one!"); 121 } 122 123 array[i] = .{ 124 .msg = msg, 125 .line = line - 1, 126 .column = column - 1, 127 }; 128 } 129 self.updates.append(.{ .src = src, .case = .{ .Error = array } }) catch unreachable; 130 } 131 132 /// Adds a subcase in which the module is updated with `src`, and 133 /// asserts that it compiles without issue 134 pub fn compiles(self: *Case, src: [:0]const u8) void { 135 self.addError(src, &[_][]const u8{}); 136 } 137 }; 138 139 pub fn addExe( 140 ctx: *TestContext, 141 name: []const u8, 142 target: std.zig.CrossTarget, 143 T: TestType, 144 ) *Case { 145 ctx.cases.append(Case{ 146 .name = name, 147 .target = target, 148 .updates = std.ArrayList(Update).init(ctx.cases.allocator), 149 .output_mode = .Exe, 150 .extension = T, 151 }) catch unreachable; 152 return &ctx.cases.items[ctx.cases.items.len - 1]; 153 } 154 155 /// Adds a test case for Zig input, producing an executable 156 pub fn exe(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case { 157 return ctx.addExe(name, target, .Zig); 158 } 159 160 /// Adds a test case for ZIR input, producing an executable 161 pub fn exeZIR(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case { 162 return ctx.addExe(name, target, .ZIR); 163 } 164 165 pub fn addObj( 166 ctx: *TestContext, 167 name: []const u8, 168 target: std.zig.CrossTarget, 169 T: TestType, 170 ) *Case { 171 ctx.cases.append(Case{ 172 .name = name, 173 .target = target, 174 .updates = std.ArrayList(Update).init(ctx.cases.allocator), 175 .output_mode = .Obj, 176 .extension = T, 177 }) catch unreachable; 178 return &ctx.cases.items[ctx.cases.items.len - 1]; 179 } 180 181 /// Adds a test case for Zig input, producing an object file 182 pub fn obj(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case { 183 return ctx.addObj(name, target, .Zig); 184 } 185 186 /// Adds a test case for ZIR input, producing an object file 187 pub fn objZIR(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget) *Case { 188 return ctx.addObj(name, target, .ZIR); 189 } 190 191 pub fn addC(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, T: TestType, standard: Module.CStandard) *Case { 192 ctx.cases.append(Case{ 193 .name = name, 194 .target = target, 195 .updates = std.ArrayList(Update).init(ctx.cases.allocator), 196 .output_mode = .Obj, 197 .extension = T, 198 .c_standard = standard, 199 }) catch unreachable; 200 return &ctx.cases.items[ctx.cases.items.len - 1]; 201 } 202 203 pub fn c11(ctx: *TestContext, name: []const u8, target: std.zig.CrossTarget, src: [:0]const u8, c: [:0]const u8) void { 204 ctx.addC(name, target, .Zig, .C11).addTransform(src, c); 205 } 206 207 pub fn addCompareOutput( 208 ctx: *TestContext, 209 name: []const u8, 210 T: TestType, 211 src: [:0]const u8, 212 expected_stdout: []const u8, 213 ) void { 214 ctx.addExe(name, .{}, T).addCompareOutput(src, expected_stdout); 215 } 216 217 /// Adds a test case that compiles the Zig source given in `src`, executes 218 /// it, runs it, and tests the output against `expected_stdout` 219 pub fn compareOutput( 220 ctx: *TestContext, 221 name: []const u8, 222 src: [:0]const u8, 223 expected_stdout: []const u8, 224 ) void { 225 return ctx.addCompareOutput(name, .Zig, src, expected_stdout); 226 } 227 228 /// Adds a test case that compiles the ZIR source given in `src`, executes 229 /// it, runs it, and tests the output against `expected_stdout` 230 pub fn compareOutputZIR( 231 ctx: *TestContext, 232 name: []const u8, 233 src: [:0]const u8, 234 expected_stdout: []const u8, 235 ) void { 236 ctx.addCompareOutput(name, .ZIR, src, expected_stdout); 237 } 238 239 pub fn addTransform( 240 ctx: *TestContext, 241 name: []const u8, 242 target: std.zig.CrossTarget, 243 T: TestType, 244 src: [:0]const u8, 245 result: [:0]const u8, 246 ) void { 247 ctx.addObj(name, target, T).addTransform(src, result); 248 } 249 250 /// Adds a test case that compiles the Zig given in `src` to ZIR and tests 251 /// the ZIR against `result` 252 pub fn transform( 253 ctx: *TestContext, 254 name: []const u8, 255 target: std.zig.CrossTarget, 256 src: [:0]const u8, 257 result: [:0]const u8, 258 ) void { 259 ctx.addTransform(name, target, .Zig, src, result); 260 } 261 262 /// Adds a test case that cleans up the ZIR source given in `src`, and 263 /// tests the resulting ZIR against `result` 264 pub fn transformZIR( 265 ctx: *TestContext, 266 name: []const u8, 267 target: std.zig.CrossTarget, 268 src: [:0]const u8, 269 result: [:0]const u8, 270 ) void { 271 ctx.addTransform(name, target, .ZIR, src, result); 272 } 273 274 pub fn addError( 275 ctx: *TestContext, 276 name: []const u8, 277 target: std.zig.CrossTarget, 278 T: TestType, 279 src: [:0]const u8, 280 expected_errors: []const []const u8, 281 ) void { 282 ctx.addObj(name, target, T).addError(src, expected_errors); 283 } 284 285 /// Adds a test case that ensures that the Zig given in `src` fails to 286 /// compile for the expected reasons, given in sequential order in 287 /// `expected_errors` in the form `:line:column: error: message`. 288 pub fn compileError( 289 ctx: *TestContext, 290 name: []const u8, 291 target: std.zig.CrossTarget, 292 src: [:0]const u8, 293 expected_errors: []const []const u8, 294 ) void { 295 ctx.addError(name, target, .Zig, src, expected_errors); 296 } 297 298 /// Adds a test case that ensures that the ZIR given in `src` fails to 299 /// compile for the expected reasons, given in sequential order in 300 /// `expected_errors` in the form `:line:column: error: message`. 301 pub fn compileErrorZIR( 302 ctx: *TestContext, 303 name: []const u8, 304 target: std.zig.CrossTarget, 305 src: [:0]const u8, 306 expected_errors: []const []const u8, 307 ) void { 308 ctx.addError(name, target, .ZIR, src, expected_errors); 309 } 310 311 pub fn addCompiles( 312 ctx: *TestContext, 313 name: []const u8, 314 target: std.zig.CrossTarget, 315 T: TestType, 316 src: [:0]const u8, 317 ) void { 318 ctx.addObj(name, target, T).compiles(src); 319 } 320 321 /// Adds a test case that asserts that the Zig given in `src` compiles 322 /// without any errors. 323 pub fn compiles( 324 ctx: *TestContext, 325 name: []const u8, 326 target: std.zig.CrossTarget, 327 src: [:0]const u8, 328 ) void { 329 ctx.addCompiles(name, target, .Zig, src); 330 } 331 332 /// Adds a test case that asserts that the ZIR given in `src` compiles 333 /// without any errors. 334 pub fn compilesZIR( 335 ctx: *TestContext, 336 name: []const u8, 337 target: std.zig.CrossTarget, 338 src: [:0]const u8, 339 ) void { 340 ctx.addCompiles(name, target, .ZIR, src); 341 } 342 343 /// Adds a test case that first ensures that the Zig given in `src` fails 344 /// to compile for the reasons given in sequential order in 345 /// `expected_errors` in the form `:line:column: error: message`, then 346 /// asserts that fixing the source (updating with `fixed_src`) isn't broken 347 /// by incremental compilation. 348 pub fn incrementalFailure( 349 ctx: *TestContext, 350 name: []const u8, 351 target: std.zig.CrossTarget, 352 src: [:0]const u8, 353 expected_errors: []const []const u8, 354 fixed_src: [:0]const u8, 355 ) void { 356 var case = ctx.addObj(name, target, .Zig); 357 case.addError(src, expected_errors); 358 case.compiles(fixed_src); 359 } 360 361 /// Adds a test case that first ensures that the ZIR given in `src` fails 362 /// to compile for the reasons given in sequential order in 363 /// `expected_errors` in the form `:line:column: error: message`, then 364 /// asserts that fixing the source (updating with `fixed_src`) isn't broken 365 /// by incremental compilation. 366 pub fn incrementalFailureZIR( 367 ctx: *TestContext, 368 name: []const u8, 369 target: std.zig.CrossTarget, 370 src: [:0]const u8, 371 expected_errors: []const []const u8, 372 fixed_src: [:0]const u8, 373 ) void { 374 var case = ctx.addObj(name, target, .ZIR); 375 case.addError(src, expected_errors); 376 case.compiles(fixed_src); 377 } 378 379 fn init() TestContext { 380 const allocator = std.heap.page_allocator; 381 return .{ .cases = std.ArrayList(Case).init(allocator) }; 382 } 383 384 fn deinit(self: *TestContext) void { 385 for (self.cases.items) |c| { 386 for (c.updates.items) |u| { 387 if (u.case == .Error) { 388 c.updates.allocator.free(u.case.Error); 389 } 390 } 391 c.updates.deinit(); 392 } 393 self.cases.deinit(); 394 self.* = undefined; 395 } 396 397 fn run(self: *TestContext) !void { 398 var progress = std.Progress{}; 399 const root_node = try progress.start("tests", self.cases.items.len); 400 defer root_node.end(); 401 402 const native_info = try std.zig.system.NativeTargetInfo.detect(std.heap.page_allocator, .{}); 403 404 for (self.cases.items) |case| { 405 std.testing.base_allocator_instance.reset(); 406 407 var prg_node = root_node.start(case.name, case.updates.items.len); 408 prg_node.activate(); 409 defer prg_node.end(); 410 411 // So that we can see which test case failed when the leak checker goes off, 412 // or there's an internal error 413 progress.initial_delay_ns = 0; 414 progress.refresh_rate_ns = 0; 415 416 const info = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, case.target); 417 try self.runOneCase(std.testing.allocator, &prg_node, case, info.target); 418 try std.testing.allocator_instance.validate(); 419 } 420 } 421 422 fn runOneCase(self: *TestContext, allocator: *Allocator, root_node: *std.Progress.Node, case: Case, target: std.Target) !void { 423 var tmp = std.testing.tmpDir(.{}); 424 defer tmp.cleanup(); 425 426 const tmp_src_path = if (case.extension == .Zig) "test_case.zig" else if (case.extension == .ZIR) "test_case.zir" else unreachable; 427 const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path); 428 defer root_pkg.destroy(); 429 430 const bin_name = try std.zig.binNameAlloc(allocator, "test_case", target, case.output_mode, null); 431 defer allocator.free(bin_name); 432 433 var module = try Module.init(allocator, .{ 434 .target = target, 435 // TODO: support tests for object file building, and library builds 436 // and linking. This will require a rework to support multi-file 437 // tests. 438 .output_mode = case.output_mode, 439 // TODO: support testing optimizations 440 .optimize_mode = .Debug, 441 .bin_file_dir = tmp.dir, 442 .bin_file_path = bin_name, 443 .root_pkg = root_pkg, 444 .keep_source_files_loaded = true, 445 .c_standard = case.c_standard, 446 }); 447 defer module.deinit(); 448 449 for (case.updates.items) |update, update_index| { 450 var update_node = root_node.start("update", 3); 451 update_node.activate(); 452 defer update_node.end(); 453 454 var sync_node = update_node.start("write", null); 455 sync_node.activate(); 456 try tmp.dir.writeFile(tmp_src_path, update.src); 457 sync_node.end(); 458 459 var module_node = update_node.start("parse/analysis/codegen", null); 460 module_node.activate(); 461 try module.makeBinFileWritable(); 462 try module.update(); 463 module_node.end(); 464 465 switch (update.case) { 466 .Transformation => |expected_output| { 467 update_node.estimated_total_items = 5; 468 var emit_node = update_node.start("emit", null); 469 emit_node.activate(); 470 var new_zir_module = try zir.emit(allocator, module); 471 defer new_zir_module.deinit(allocator); 472 emit_node.end(); 473 474 var write_node = update_node.start("write", null); 475 write_node.activate(); 476 var out_zir = std.ArrayList(u8).init(allocator); 477 defer out_zir.deinit(); 478 try new_zir_module.writeToStream(allocator, out_zir.outStream()); 479 write_node.end(); 480 481 var test_node = update_node.start("assert", null); 482 test_node.activate(); 483 defer test_node.end(); 484 const label = if (case.c_standard) |_| "C" else "ZIR"; 485 if (expected_output.len != out_zir.items.len) { 486 std.debug.warn("{}\nTransformed {} length differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); 487 std.process.exit(1); 488 } 489 for (expected_output) |e, i| { 490 if (out_zir.items[i] != e) { 491 if (expected_output.len != out_zir.items.len) { 492 std.debug.warn("{}\nTransformed {} differs:\n================\nExpected:\n================\n{}\n================\nFound:\n================\n{}\n================\nTest failed.\n", .{ case.name, label, expected_output, out_zir.items }); 493 std.process.exit(1); 494 } 495 } 496 } 497 }, 498 .Error => |e| { 499 var test_node = update_node.start("assert", null); 500 test_node.activate(); 501 defer test_node.end(); 502 var handled_errors = try allocator.alloc(bool, e.len); 503 defer allocator.free(handled_errors); 504 for (handled_errors) |*h| { 505 h.* = false; 506 } 507 var all_errors = try module.getAllErrorsAlloc(); 508 defer all_errors.deinit(allocator); 509 for (all_errors.list) |a| { 510 for (e) |ex, i| { 511 if (a.line == ex.line and a.column == ex.column and std.mem.eql(u8, ex.msg, a.msg)) { 512 handled_errors[i] = true; 513 break; 514 } 515 } else { 516 std.debug.warn("{}\nUnexpected error:\n================\n:{}:{}: error: {}\n================\nTest failed.\n", .{ case.name, a.line + 1, a.column + 1, a.msg }); 517 std.process.exit(1); 518 } 519 } 520 521 for (handled_errors) |h, i| { 522 if (!h) { 523 const er = e[i]; 524 std.debug.warn("{}\nDid not receive error:\n================\n{}:{}: {}\n================\nTest failed.\n", .{ case.name, er.line, er.column, er.msg }); 525 std.process.exit(1); 526 } 527 } 528 }, 529 .Execution => |expected_stdout| { 530 update_node.estimated_total_items = 4; 531 var exec_result = x: { 532 var exec_node = update_node.start("execute", null); 533 exec_node.activate(); 534 defer exec_node.end(); 535 536 try module.makeBinFileExecutable(); 537 538 const exe_path = try std.fmt.allocPrint(allocator, "." ++ std.fs.path.sep_str ++ "{}", .{bin_name}); 539 defer allocator.free(exe_path); 540 541 break :x try std.ChildProcess.exec(.{ 542 .allocator = allocator, 543 .argv = &[_][]const u8{exe_path}, 544 .cwd_dir = tmp.dir, 545 }); 546 }; 547 var test_node = update_node.start("test", null); 548 test_node.activate(); 549 defer test_node.end(); 550 551 defer allocator.free(exec_result.stdout); 552 defer allocator.free(exec_result.stderr); 553 switch (exec_result.term) { 554 .Exited => |code| { 555 if (code != 0) { 556 std.debug.warn("elf file exited with code {}\n", .{code}); 557 return error.BinaryBadExitCode; 558 } 559 }, 560 else => return error.BinaryCrashed, 561 } 562 if (!std.mem.eql(u8, expected_stdout, exec_result.stdout)) { 563 std.debug.panic( 564 "update index {}, mismatched stdout\n====Expected (len={}):====\n{}\n====Actual (len={}):====\n{}\n========\n", 565 .{ update_index, expected_stdout.len, expected_stdout, exec_result.stdout.len, exec_result.stdout }, 566 ); 567 } 568 }, 569 } 570 } 571 } 572 };