From ae3d95fc8dc10c0902dee0605421cec2dbd316b0 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Tue, 21 Jan 2025 08:46:12 -0500 Subject: [PATCH 1/5] x86_64: rewrite scalar float equality comparisons --- src/arch/x86_64/CodeGen.zig | 952 +++++++++++++++++++++++++++------- src/arch/x86_64/Encoding.zig | 5 +- src/arch/x86_64/Mir.zig | 8 +- src/arch/x86_64/bits.zig | 6 +- src/arch/x86_64/encodings.zig | 6 + test/behavior/x86_64/math.zig | 47 +- 6 files changed, 838 insertions(+), 186 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index a049217867..b599343730 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -150,7 +150,7 @@ const Owner = union(enum) { }; const MaskKind = enum(u1) { sign, all }; -const MaskInfo = packed struct { kind: MaskKind, inverted: bool, scalar: Memory.Size }; +const MaskInfo = packed struct { kind: MaskKind, inverted: bool = false, scalar: Memory.Size }; pub const MCValue = union(enum) { /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. @@ -2657,10 +2657,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } else { // hack around Sema OPV bugs res[0] = ops[0]; } - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .ptr_sub => |air_tag| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; @@ -2792,17 +2789,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { // hack around Sema OPV bugs res[0] = ops[0]; } - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .alloc => if (use_old) try cg.airAlloc(inst) else { const ty = air_datas[@intFromEnum(inst)].ty; - var slot = try cg.tempInit(ty, .{ .lea_frame = .{ + const slot = try cg.tempInit(ty, .{ .lea_frame = .{ .index = try cg.allocMemPtr(inst), } }); - try slot.moveTo(inst, cg); + try slot.finish(inst, &.{}, &.{}, cg); }, .inferred_alloc, .inferred_alloc_comptime => unreachable, .ret_ptr => if (use_old) try cg.airRetPtr(inst) else { @@ -2818,7 +2812,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { break :slot slot; }, }; - try slot.moveTo(inst, cg); + try slot.finish(inst, &.{}, &.{}, cg); }, .assembly => try cg.airAsm(inst), .bit_and, .bit_or, .xor, .bool_and, .bool_or => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else { @@ -3154,10 +3148,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }), else => |e| return e, }; - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .not => |air_tag| if (use_old) try cg.airUnOp(inst, air_tag) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -4207,10 +4198,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }), else => |e| return e, }; - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .block => if (use_old) try cg.airBlock(inst) else { @@ -4250,13 +4238,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .index = .ret_addr, } }); while (try slot.toRegClass(true, .general_purpose, cg)) {} - try slot.moveTo(inst, cg); + try slot.finish(inst, &.{}, &.{}, cg); }, .frame_addr => if (use_old) try cg.airFrameAddress(inst) else { - var slot = try cg.tempInit(.usize, .{ .lea_frame = .{ + const slot = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = .base_ptr, } }); - try slot.moveTo(inst, cg); + try slot.finish(inst, &.{}, &.{}, cg); }, .call => try cg.airCall(inst, .auto, .{ .safety = true }), .call_always_tail => try cg.airCall(inst, .always_tail, .{ .safety = true }), @@ -6986,10 +6974,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }), else => |e| return e, }; - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: { @@ -8394,6 +8379,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .dst_constraints = .{.{ .bool_vec = .byte }}, + .src_constraints = .{ .any_scalar_int, .any_scalar_int }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -8550,6 +8536,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ .any_scalar_int, .any_scalar_int }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -8710,6 +8697,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .{ .required_features = .{ .@"64bit", null, null, null }, .dst_constraints = .{.{ .bool_vec = .qword }}, + .src_constraints = .{ .any_scalar_int, .any_scalar_int }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -8784,6 +8772,72 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._r, .sh, .tmp3d, .si(3), ._, ._ }, .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ }, } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_exact_float = .{ .of = .qword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem } }, + .{ .src = .{ .sse, .mem } }, + .{ .src = .{ .mem, .sse } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .kind = .all, + .scalar = .dword, + } } }}, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, + .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .si(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_exact_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_exact_float = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .mem, .mem } }, + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .kind = .all, + .scalar = .dword, + } } }}, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, + .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .si(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, } }, }) catch |err| switch (err) { error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ @@ -8797,10 +8851,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .gte => unreachable, .gt => unreachable, } - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ extra.lhs, extra.rhs }, &ops, cg); }, .abs => |air_tag| if (use_old) try cg.airAbs(inst) else fallback: { @@ -9836,10 +9887,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .required_features = .{ .x87, null, null, null }, .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .tbyte } }, .any }, .patterns = &.{ + .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_x87, .none } }, }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .x87 } }}, - .clobbers = .{ .st = 1 }, .each = .{ .once = &.{ .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, .{ ._, .f_, .abs, ._, ._, ._, ._ }, @@ -9967,10 +10026,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }), else => |e| return e, }; - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .cmp_lt, @@ -10166,10 +10222,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }), else => |e| return e, }; - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .cmp_eq, .cmp_eq_optimized, @@ -10182,7 +10235,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }) else fallback: { const bin_op = air_datas[@intFromEnum(inst)].bin_op; const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu); - if (cg.intInfo(scalar_ty) == null) break :fallback try cg.airCmp(inst, switch (air_tag) { + if (cg.intInfo(scalar_ty) == null and cg.floatBits(scalar_ty) == null) break :fallback try cg.airCmp(inst, switch (air_tag) { else => unreachable, .cmp_eq, .cmp_eq_optimized => .eq, .cmp_neq, .cmp_neq_optimized => .neq, @@ -10424,6 +10477,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10481,6 +10538,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + .{ .remainder_int = .{ .of = .yword, .is = .yword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10506,6 +10567,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10531,6 +10596,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10556,6 +10625,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_int = .{ .of = .xword, .is = .xword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10584,6 +10657,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10611,6 +10688,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._, .cmp, .tmp0d, .si(0xff), ._, ._ }, } }, }, .{ + .src_constraints = .{ + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + .{ .remainder_int = .{ .of = .qword, .is = .qword } }, + }, .patterns = &.{ .{ .src = .{ .to_mem, .to_mem } }, }, @@ -10634,6 +10715,443 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, .{ ._, ._, .@"test", .tmp1p, .tmp1p, ._, ._ }, } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .vp_, .unpcklwd, .tmp0x, .src0x, .src1x, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, .v_, .movshdup, .tmp1x, .tmp0x, ._, ._ }, + .{ ._, .v_ss, .ucomi, .tmp0x, .tmp1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .word }, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, + }, + .call_frame = .{ .size = 0, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = cc }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_ss, .ucomi, .src0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._ss, .ucomi, .src0x, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .dword }, .{ .float = .word } }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, + }, + .call_frame = .{ .size = 0, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqsf2", + .ne => "__nesf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = cc }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .v_sd, .ucomi, .src0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .{ .type = .f16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._sd, .ucomi, .src0x, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .qword }, .{ .float = .qword } }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, + }, + .call_frame = .{ .size = 0, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqdf2", + .ne => "__nedf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = cc }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .mem, .mem } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp0t, .tmp1t, ._, ._ }, + .{ ._, .f_p, .st, .tmp1t, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87 } }, + .{ .src = .{ .to_x87, .to_x87 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp0t, .src1t, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .mem, .mem } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .mem, .mem } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u8, .kind = .{ .reg = .ah } }, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = cc }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b0_1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .mem, .mem } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src1t, ._, ._, ._ }, + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sahf, .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87 } }, + .{ .src = .{ .to_x87, .to_x87 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .x87, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87 } }, + .{ .src = .{ .to_x87, .to_x87 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u8, .kind = .{ .reg = .ah } }, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = cc }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp3b, .si(0b0_1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ .{ .float = .tbyte }, .{ .float = .tbyte } }, + .patterns = &.{ + .{ .src = .{ .to_x87, .mem }, .commute = .{ 0, 1 } }, + .{ .src = .{ .mem, .to_x87 } }, + .{ .src = .{ .to_x87, .to_x87 } }, + }, + .extra_temps = .{ + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u16, .kind = .{ .reg = .ax } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = switch (cc) { + else => unreachable, + .e => .z_and_np, + .ne => .nz_or_p, + } }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, + .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, + .{ ._, ._, .sahf, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword } }, + .patterns = &.{ + .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, + }, + .call_frame = .{ .size = 0, .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .cc = cc }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, + } }, } }, }) catch |err| switch (err) { error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ @@ -10644,10 +11162,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }), else => |e| return e, }; - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .cond_br => try cg.airCondBr(inst), @@ -10718,9 +11233,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .fromSize(opt_child_abi_size) }), .u(0), ); - var is_null = try cg.tempInit(.bool, .{ .eflags = .e }); - try ops[0].die(cg); - try is_null.moveTo(inst, cg); + const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); + try is_null.finish(inst, &.{un_op}, &ops, cg); }, .is_non_null_ptr => if (use_old) try cg.airIsNonNullPtr(inst) else { const un_op = air_datas[@intFromEnum(inst)].un_op; @@ -10741,9 +11255,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .fromSize(opt_child_abi_size) }), .u(0), ); - var is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); - try ops[0].die(cg); - try is_non_null.moveTo(inst, cg); + const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); + try is_non_null.finish(inst, &.{un_op}, &ops, cg); }, .is_err_ptr => if (use_old) try cg.airIsErrPtr(inst) else { const un_op = air_datas[@intFromEnum(inst)].un_op; @@ -10759,9 +11272,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), .u(0), ); - var is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); - try ops[0].die(cg); - try is_err.moveTo(inst, cg); + const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); + try is_err.finish(inst, &.{un_op}, &ops, cg); }, .is_non_err_ptr => if (use_old) try cg.airIsNonErrPtr(inst) else { const un_op = air_datas[@intFromEnum(inst)].un_op; @@ -10777,9 +11289,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), .u(0), ); - var is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); - try ops[0].die(cg); - try is_non_err.moveTo(inst, cg); + const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); + try is_non_err.finish(inst, &.{un_op}, &ops, cg); }, .load => if (use_old) try cg.airLoad(inst) else fallback: { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -10790,26 +11301,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { (ptr_info.flags.vector_index == .none or val_ty.toIntern() == .bool_type)) break :fallback try cg.airLoad(inst); var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - var res = try ops[0].load(val_ty, .{ + const res = try ops[0].load(val_ty, .{ .disp = switch (ptr_info.flags.vector_index) { .none => 0, .runtime => unreachable, else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), }, }, cg); - for (ops) |op| if (op.index != res.index) try op.die(cg); - try res.moveTo(inst, cg); + try res.finish(inst, &.{ty_op.operand}, &ops, cg); }, .int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else { const un_op = air_datas[@intFromEnum(inst)].un_op; var ops = try cg.tempsFromOperands(inst, .{un_op}); try ops[0].toSlicePtr(cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{un_op}, &ops, cg); }, .int_from_bool => if (use_old) try cg.airIntFromBool(inst) else { const un_op = air_datas[@intFromEnum(inst)].un_op; - var ops = try cg.tempsFromOperands(inst, .{un_op}); - try ops[0].moveTo(inst, cg); + const ops = try cg.tempsFromOperands(inst, .{un_op}); + try ops[0].finish(inst, &.{un_op}, &ops, cg); }, .ret => try cg.airRet(inst, false), .ret_safe => try cg.airRet(inst, true), @@ -10848,8 +11358,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unreach => {}, .optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].moveTo(inst, cg); + const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .optional_payload_ptr_set => if (use_old) try cg.airOptionalPayloadPtrSet(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -10864,7 +11374,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try has_value.die(cg); try ops[0].toOffset(-opt_child_abi_size, cg); } - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .unwrap_errunion_payload_ptr => if (use_old) try cg.airUnwrapErrUnionPayloadPtr(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -10873,7 +11383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); try ops[0].toOffset(eu_pl_off, cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .unwrap_errunion_err_ptr => if (use_old) try cg.airUnwrapErrUnionErrPtr(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -10882,9 +11392,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); try ops[0].toOffset(eu_err_off, cg); - var err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg); - try ops[0].die(cg); - try err.moveTo(inst, cg); + const err = try ops[0].load(eu_ty.errorUnionSet(zcu), .{}, cg); + try err.finish(inst, &.{ty_op.operand}, &ops, cg); }, .errunion_payload_ptr_set => if (use_old) try cg.airErrUnionPayloadPtrSet(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -10899,7 +11408,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try ops[0].store(&no_err, .{}, cg); try no_err.die(cg); try ops[0].toOffset(eu_pl_off - eu_err_off, cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .struct_field_ptr => if (use_old) try cg.airStructFieldPtr(inst) else { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; @@ -10910,7 +11419,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { ty_pl.ty.toType(), extra.field_index, ), cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{extra.struct_operand}, &ops, cg); }, .struct_field_ptr_index_0, .struct_field_ptr_index_1, @@ -10936,7 +11445,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .struct_field_ptr_index_3 => 3, }, ), cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .struct_field_val => if (use_old) try cg.airStructFieldVal(inst) else fallback: { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; @@ -10953,8 +11462,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try ops[0].read(field_ty, .{ .disp = field_off }, cg) else try cg.tempInit(field_ty, .none); - for (ops) |op| if (op.index != res.index) try op.die(cg); - try res.moveTo(inst, cg); + try res.finish(inst, &.{extra.struct_operand}, &ops, cg); }, .set_union_tag => if (use_old) try cg.airSetUnionTag(inst) else { const bin_op = air_datas[@intFromEnum(inst)].bin_op; @@ -10965,7 +11473,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { if (union_layout.tag_size > 0) try ops[0].store(&ops[1], .{ .disp = @intCast(union_layout.tagOffset()), }, cg); - for (ops) |op| try op.die(cg); + const res = try cg.tempInit(.void, .none); + try res.finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .get_union_tag => if (use_old) try cg.airGetUnionTag(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -10973,42 +11482,41 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); const union_layout = union_ty.unionGetLayout(zcu); assert(union_layout.tag_size > 0); - var res = try ops[0].read(ty_op.ty.toType(), .{ + const res = try ops[0].read(ty_op.ty.toType(), .{ .disp = @intCast(union_layout.tagOffset()), }, cg); - for (ops) |op| if (op.index != res.index) try op.die(cg); - try res.moveTo(inst, cg); + try res.finish(inst, &.{ty_op.operand}, &ops, cg); }, .slice => if (use_old) try cg.airSlice(inst) else { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); try ops[0].toPair(&ops[1], cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .slice_len => if (use_old) try cg.airSliceLen(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); try ops[0].toSliceLen(cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .slice_ptr => if (use_old) try cg.airSlicePtr(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); try ops[0].toSlicePtr(cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .ptr_slice_len_ptr => if (use_old) try cg.airPtrSliceLenPtr(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); try ops[0].toOffset(8, cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .ptr_slice_ptr_ptr => if (use_old) try cg.airPtrSlicePtrPtr(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); try ops[0].toOffset(0, cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .slice_elem_val, .ptr_elem_val => |air_tag| if (use_old) switch (air_tag) { else => unreachable, @@ -11139,10 +11647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { // hack around Sema OPV bugs res[0] = try cg.tempInit(res_ty, .none); } - for (ops) |op| for (res) |r| { - if (op.index == r.index) break; - } else try op.die(cg); - try res[0].moveTo(inst, cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .slice_elem_ptr, .ptr_elem_ptr => |air_tag| if (use_old) switch (air_tag) { else => unreachable, @@ -11195,8 +11700,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }); } - try ops[1].die(cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, .array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; @@ -11205,7 +11709,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), }); try ops[0].toPair(&len, cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .error_set_has_value => return cg.fail("TODO implement error_set_has_value", .{}), .union_init => if (use_old) try cg.airUnionInit(inst) else { @@ -11228,8 +11732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try res.write(&ops[0], .{ .disp = @intCast(union_layout.payloadOffset()), }, cg); - try ops[0].die(cg); - try res.moveTo(inst, cg); + try res.finish(inst, &.{extra.init}, &ops, cg); }, .field_parent_ptr => if (use_old) try cg.airFieldParentPtr(inst) else { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; @@ -11240,7 +11743,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { cg.typeOf(extra.field_ptr), extra.field_index, ), cg); - try ops[0].moveTo(inst, cg); + try ops[0].finish(inst, &.{extra.field_ptr}, &ops, cg); }, .is_named_enum_value => return cg.fail("TODO implement is_named_enum_value", .{}), @@ -11274,8 +11777,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .addrspace_cast => { const ty_op = air_datas[@intFromEnum(inst)].ty_op; - var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); - try ops[0].moveTo(inst, cg); + const ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg); }, .save_err_return_trace_index => { @@ -21474,11 +21977,16 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { const ret_ty = self.fn_type.fnReturnType(zcu); switch (self.ret_mcv.short) { .none => {}, - .register, - .register_pair, - .register_triple, - .register_quadruple, - => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }), + .register => |reg| { + const reg_lock = self.register_manager.lockRegAssumeUnused(reg); + defer self.register_manager.unlockReg(reg_lock); + try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); + }, + inline .register_pair, .register_triple, .register_quadruple => |regs| { + const reg_locks = self.register_manager.lockRegsAssumeUnused(regs.len, regs); + defer for (reg_locks) |reg_lock| self.register_manager.unlockReg(reg_lock); + try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }); + }, .indirect => |reg_off| { try self.register_manager.getReg(reg_off.reg, null); const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg); @@ -29772,6 +30280,7 @@ const Temp = struct { cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; result_temp_index.tracking(cg).* = .init(result); first_temp.* = result_temp; + second_temp.* = result_temp; } fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void { @@ -30248,7 +30757,20 @@ const Temp = struct { try cg.asmOpOnly(.{ .@"rep _sb", .sto }); } - fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void { + fn finish( + temp: Temp, + inst: Air.Inst.Index, + op_refs: []const Air.Inst.Ref, + op_temps: []const Temp, + cg: *CodeGen, + ) !void { + const tomb_bits = cg.liveness.getTombBits(inst); + for (0.., op_refs, op_temps) |op_index, op_ref, op_temp| { + if (op_temp.index != temp.index) try op_temp.die(cg); + if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (cg.reused_operands.isSet(op_index)) continue; + try cg.processDeath(op_ref.toIndexAllowNone() orelse continue); + } if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) { .ref, .err_ret_trace => { const result = try cg.allocRegOrMem(inst, true); @@ -30529,83 +31051,114 @@ const Select = struct { }; switch (mir_tag[0]) { .f_ => switch (mir_tag[1]) { - .abs, .st => {}, + .@"2xm1", + .abs, + .add, + .chs, + .clex, + .com, + .comi, + .cos, + .div, + .divr, + .free, + .mul, + .nop, + .prem, + .rndint, + .scale, + .sin, + .sqrt, + .st, + .sub, + .subr, + .tst, + .ucom, + .ucomi, + .wait, + .xam, + .xch, + => {}, + .init, .save => s.top = 0, + .ld, .ptan, .sincos, .xtract => s.top -%= 1, + .patan, .yl2x => s.top +%= 1, + .rstor => unreachable, + else => unreachable, + }, + .f_1 => switch (mir_tag[1]) { .ld => s.top -%= 1, - else => { - const fixes = @tagName(mir_tag[0]); - const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; - std.debug.panic("{s}: {s}{s}{s}\n", .{ - @src().fn_name, - fixes[0..fixes_blank], - @tagName(mir_tag[1]), - fixes[fixes_blank + 1 ..], - }); - }, + .prem => {}, + else => unreachable, }, - .f_p => switch (mir_tag[1]) { - .st => s.top +%= 1, - else => { - const fixes = @tagName(mir_tag[0]); - const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; - std.debug.panic("{s}: {s}{s}{s}\n", .{ - @src().fn_name, - fixes[0..fixes_blank], - @tagName(mir_tag[1]), - fixes[fixes_blank + 1 ..], - }); - }, + .f_b, .f_be, .f_e, .f_nb, .f_nbe, .f_ne, .f_nu, .f_u => switch (mir_tag[1]) { + .cmov => {}, + else => unreachable, }, - .f_1, - => switch (mir_tag[1]) { - .ld => s.top -%= 1, - else => { - const fixes = @tagName(mir_tag[0]); - const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; - std.debug.panic("{s}: {s}{s}{s}\n", .{ - @src().fn_name, - fixes[0..fixes_blank], - @tagName(mir_tag[1]), - fixes[fixes_blank + 1 ..], - }); - }, + .f_cw, .f_env, .f_sw => switch (mir_tag[1]) { + .ld, .st => {}, + else => unreachable, }, - .f_l2e, - .f_l2t, - .f_lg2, - .f_ln2, - .f_pi, - .f_z, - => switch (mir_tag[1]) { + .f_p1 => switch (mir_tag[1]) { + .yl2x => s.top +%= 1, + else => unreachable, + }, + .fb_ => switch (mir_tag[1]) { .ld => s.top -%= 1, else => unreachable, }, - .f_b, - .f_be, - .f_cw, - .f_e, - .f_env, - .f_nb, - .f_nbe, - .f_ne, - .f_nu, - .f_p1, - .f_pp, - .f_sw, - .f_u, - .fb_, - .fb_p, - .fi_, - .fi_p, - .fn_, - .fn_cw, - .fn_env, - .fn_sw, - => {}, + .fb_p => switch (mir_tag[1]) { + .st => s.top +%= 1, + else => unreachable, + }, + .fi_ => switch (mir_tag[1]) { + .add, .com, .div, .divr, .mul, .st, .stt, .sub, .subr => {}, + .ld => s.top -%= 1, + else => unreachable, + }, + .fi_p => switch (mir_tag[1]) { + .com, .st => s.top +%= 1, + else => unreachable, + }, + .fn_ => switch (mir_tag[1]) { + .clex => {}, + .init, .save => s.top = 0, + else => unreachable, + }, + .fn_cw, .fn_env, .fn_sw => switch (mir_tag[1]) { + .st => {}, + else => unreachable, + }, .f_cstp => switch (mir_tag[1]) { .de => s.top -%= 1, .in => s.top +%= 1, else => unreachable, }, + .f_l2e, .f_l2t, .f_lg2, .f_ln2, .f_pi, .f_z => switch (mir_tag[1]) { + .ld => s.top -%= 1, + else => unreachable, + }, + .f_p => switch (mir_tag[1]) { + .add, .com, .comi, .div, .divr, .mul, .st, .sub, .subr, .ucom, .ucomi => s.top +%= 1, + else => { + const fixes = @tagName(mir_tag[0]); + const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; + std.debug.panic("{s}: {s}{s}{s}\n", .{ + @src().fn_name, + fixes[0..fixes_blank], + @tagName(mir_tag[1]), + fixes[fixes_blank + 1 ..], + }); + }, + }, + .f_pp => switch (mir_tag[1]) { + .com, .ucom => s.top +%= 2, + else => unreachable, + }, + .fx_ => switch (mir_tag[1]) { + .rstor => unreachable, + .save => {}, + else => unreachable, + }, else => {}, } } @@ -30620,9 +31173,13 @@ const Select = struct { dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any), src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any), patterns: []const Select.Pattern, + call_frame: packed struct(u16) { size: u10, alignment: InternPool.Alignment } = .{ .size = 0, .alignment = .none }, extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused), dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused), - clobbers: struct { eflags: bool = false, st: u3 = 0 } = .{}, + clobbers: packed struct { + eflags: bool = false, + caller_preserved: enum(u2) { none, ccc, zigcc } = .none, + } = .{}, each: union(enum) { once: []const Instruction, }, @@ -30649,6 +31206,7 @@ const Select = struct { scalar_signed_int: Memory.Size, scalar_unsigned_int: Memory.Size, scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size }, + float: Memory.Size, scalar_float: struct { of: Memory.Size, is: Memory.Size }, scalar_exact_float: struct { of: Memory.Size, is: Memory.Size }, multiple_scalar_int: struct { of: Memory.Size, is: Memory.Size }, @@ -30716,6 +31274,7 @@ const Select = struct { of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else false, + .float => |size| if (cg.floatBits(ty)) |float_bits| size.bitSize(cg.target) == float_bits else false, .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) >= float_bits else false, .scalar_exact_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and @@ -30787,13 +31346,14 @@ const Select = struct { src: [2]Src, commute: struct { u8, u8 } = .{ 0, 0 }, - const Src = enum { + const Src = union(enum) { none, any, imm8, imm16, imm32, simm32, + to_reg: Register, mem, to_mem, mut_mem, @@ -30846,6 +31406,7 @@ const Select = struct { .mem => temp.tracking(cg).short.isMemory(), .to_mem, .to_mut_mem => true, .mut_mem => temp.isMut(cg) and temp.tracking(cg).short.isMemory(), + .to_reg => true, .gpr => temp.typeOf(cg).abiSize(cg.pt.zcu) <= 8 and switch (temp.tracking(cg).short) { .register => |reg| reg.class() == .general_purpose, .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and reg_off.off == 0, @@ -30920,6 +31481,7 @@ const Select = struct { .none => unreachable, .any, .imm8, .imm16, .imm32, .simm32 => false, .mem, .to_mem, .mut_mem, .to_mut_mem => try temp.toBase(cg), + .to_reg => |reg| try temp.toReg(reg, cg), .gpr, .to_gpr => try temp.toRegClass(false, .general_purpose, cg), .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg), .x87, .to_x87 => try temp.toRegClass(false, .x87, cg), @@ -30954,6 +31516,7 @@ const Select = struct { ref: Select.Operand.Ref, ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, mut_reg: struct { ref: Select.Operand.Ref, rc: Register.Class }, + symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 }, const ConstInfo = struct { ref: Select.Operand.Ref, vectorize: bool = false }; @@ -31048,6 +31611,11 @@ const Select = struct { }; return try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc)); }, + .symbol => |symbol| if (cg.bin_file.cast(.elf)) |elf_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ + .sym_index = try elf_file.getGlobalSymbol(symbol.name, symbol.lib), + } }) else if (cg.bin_file.cast(.macho)) |macho_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ + .sym_index = try macho_file.getGlobalSymbol(symbol.name, symbol.lib), + } }) else cg.fail("external symbols unimplemented for {s}", .{@tagName(cg.bin_file.tag)}), }; } }; @@ -31566,6 +32134,7 @@ const Select = struct { } }, else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) }, .register => |reg| .{ .reg = s.lowerReg(registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8)))) }, + .lea_symbol => |sym_off| .{ .imm = .rel(sym_off) }, }, .simm => .{ .imm = .s(op.adjustedImm(i32, s)) }, .uimm => .{ .imm = .u(@bitCast(op.adjustedImm(i64, s))) }, @@ -31601,6 +32170,7 @@ fn select( src_temps: []Temp, cases: []const Select.Case, ) !void { + @setEvalBranchQuota(33_600); cases: for (cases) |case| { for (case.required_features) |required_feature| if (required_feature) |feature| if (!cg.hasFeature(feature)) continue :cases; for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases; @@ -31613,6 +32183,16 @@ fn select( for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; if (std.debug.runtime_safety) for (pattern.src[src_temps.len..]) |src_pattern| assert(src_pattern == .none); + if (case.call_frame.alignment != .none) { + const frame_allocs_slice = cg.frame_allocs.slice(); + const stack_frame_size = + &frame_allocs_slice.items(.abi_size)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_size.* = @max(stack_frame_size.*, case.call_frame.size); + const stack_frame_align = + &frame_allocs_slice.items(.abi_align)[@intFromEnum(FrameIndex.call_frame)]; + stack_frame_align.* = stack_frame_align.max(case.call_frame.alignment); + } + var s: Select = .{ .cg = cg, .temps = undefined, @@ -31623,10 +32203,19 @@ fn select( const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; - for (0..case.clobbers.st -| 1) |i| { - const tracked_index: RegisterManager.TrackedIndex = @intCast(RegisterManager.indexOfKnownRegIntoTracked(.st6).? - i); - try cg.register_manager.getRegIndex(tracked_index, null); - _ = cg.register_manager.lockRegIndexAssumeUnused(tracked_index); + caller_preserved: { + switch (switch (case.clobbers.caller_preserved) { + .none => break :caller_preserved, + .ccc => cg.target.cCallingConvention().?, + .zigcc => .auto, + }) { + else => unreachable, + inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| { + const tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse continue; + try cg.register_manager.getRegIndex(tracked_index, null); + assert(cg.register_manager.lockRegIndexAssumeUnused(tracked_index).tracked_index == tracked_index); + }, + } } @memcpy(src_slots[0..src_temps.len], src_temps); @@ -31653,9 +32242,20 @@ fn select( } assert(s.top == 0); - for (0..case.clobbers.st -| 1) |i| cg.register_manager.unlockReg(.{ - .tracked_index = @intCast(RegisterManager.indexOfKnownRegIntoTracked(.st6).? - i), - }); + caller_preserved: { + switch (switch (case.clobbers.caller_preserved) { + .none => break :caller_preserved, + .ccc => cg.target.cCallingConvention().?, + .zigcc => .auto, + }) { + else => unreachable, + inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| { + cg.register_manager.unlockReg(.{ + .tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse continue, + }); + }, + } + } for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, &s); for (case.extra_temps, tmp_slots) |spec, temp| if (spec.kind != .unused) try temp.die(cg); return; diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig index 304bef6359..3ee548f12a 100644 --- a/src/arch/x86_64/Encoding.zig +++ b/src/arch/x86_64/Encoding.zig @@ -361,7 +361,7 @@ pub const Mnemonic = enum { addps, addss, andps, andnps, - cmpps, cmpss, + cmpps, cmpss, comiss, cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si, divps, divss, fxrstor, fxrstor64, fxsave, fxsave64, @@ -386,7 +386,7 @@ pub const Mnemonic = enum { andpd, andnpd, cmppd, //cmpsd, - comisd, comiss, + comisd, cvtdq2pd, cvtdq2ps, cvtpd2dq, cvtpd2pi, cvtpd2ps, cvtpi2pd, cvtps2dq, cvtps2pd, cvtsd2si, cvtsd2ss, cvtsi2sd, cvtss2sd, cvttpd2dq, cvttpd2pi, cvttps2dq, cvttsd2si, @@ -504,6 +504,7 @@ pub const Mnemonic = enum { vstmxcsr, vsubpd, vsubps, vsubsd, vsubss, vtestpd, vtestps, + vucomisd, vucomiss, vxorpd, vxorps, // F16C vcvtph2ps, vcvtps2ph, diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 5d86b08d70..b4d0ed94e6 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -354,6 +354,8 @@ pub const Inst = struct { fn_env, /// Float No Wait ___ status word fn_sw, + /// Float Extended ___ + fx_, /// ___ in 32-bit and Compatibility Mode _32, @@ -817,8 +819,10 @@ pub const Inst = struct { /// Round to integer rndint, /// Restore x87 FPU state + /// Restore x87 FPU, MMX, XMM, and MXCSR state rstor, /// Store x87 FPU state + /// Save x87 FPU, MMX technology, and MXCSR state save, /// Scale scale, @@ -923,10 +927,6 @@ pub const Inst = struct { /// Extract doubleword /// Extract quadword extr, - /// Restore x87 FPU, MMX, XMM, and MXCSR state - fxrstor, - /// Save x87 FPU, MMX technology, and MXCSR state - fxsave, /// Insert byte /// Insert word /// Insert doubleword diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 1ef9653386..9ab90e9881 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -366,7 +366,6 @@ pub const Register = enum(u8) { @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), - @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah) - 4, else => unreachable, // zig fmt: on }; @@ -385,7 +384,10 @@ pub const Register = enum(u8) { } pub fn to8(reg: Register) Register { - return @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.al)); + return switch (@intFromEnum(reg)) { + else => @enumFromInt(@intFromEnum(reg) - reg.gpBase() + @intFromEnum(Register.al)), + @intFromEnum(Register.ah)...@intFromEnum(Register.bh) => reg, + }; } fn sseBase(reg: Register) u7 { diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig index 389317d2ae..9357549a5a 100644 --- a/src/arch/x86_64/encodings.zig +++ b/src/arch/x86_64/encodings.zig @@ -437,6 +437,7 @@ pub const table = [_]Entry{ .{ .jmp, .d, &.{ .rel32 }, &.{ 0xe9 }, 0, .none, .none }, .{ .jmp, .m, &.{ .rm64 }, &.{ 0xff }, 4, .none, .none }, + .{ .lahf, .z, &.{}, &.{ 0x9f }, 0, .none, .@"32bit" }, .{ .lahf, .z, &.{}, &.{ 0x9f }, 0, .none, .sahf }, .{ .lar, .rm, &.{ .r16, .rm16 }, &.{ 0x0f, 0x02 }, 0, .none, .none }, @@ -744,6 +745,7 @@ pub const table = [_]Entry{ .{ .rsm, .z, &.{}, &.{ 0x0f, 0xaa }, 0, .none, .none }, + .{ .sahf, .z, &.{}, &.{ 0x9e }, 0, .none, .@"32bit" }, .{ .sahf, .z, &.{}, &.{ 0x9e }, 0, .none, .sahf }, .{ .sal, .m1, &.{ .rm8, .unity }, &.{ 0xd0 }, 4, .none, .none }, @@ -2275,6 +2277,10 @@ pub const table = [_]Entry{ .{ .vtestpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_128_w0, .avx }, .{ .vtestpd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x0f }, 0, .vex_256_w0, .avx }, + .{ .vucomisd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x2e }, 0, .vex_lig_wig, .avx }, + + .{ .vucomiss, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x0f, 0x2e }, 0, .vex_lig_wig, .avx }, + .{ .vxorpd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_128_wig, .avx }, .{ .vxorpd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x57 }, 0, .vex_256_wig, .avx }, diff --git a/test/behavior/x86_64/math.zig b/test/behavior/x86_64/math.zig index 9072461a6b..ed33e9efd0 100644 --- a/test/behavior/x86_64/math.zig +++ b/test/behavior/x86_64/math.zig @@ -17,8 +17,23 @@ const Sse = if (std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) else @Vector(16, u8); -inline fn sign(rhs: anytype) bool { - return @call(.always_inline, math.signbit, .{rhs}); +inline fn sign(rhs: anytype) switch (@typeInfo(@TypeOf(rhs))) { + else => bool, + .vector => |vector| @Vector(vector.len, bool), +} { + switch (@typeInfo(@TypeOf(rhs))) { + else => return @as(@Type(.{ .int = .{ + .signedness = .signed, + .bits = @bitSizeOf(@TypeOf(rhs)), + } }), @bitCast(rhs)) < 0, + .vector => |vector| { + const V = @Vector(vector.len, @Type(.{ .int = .{ + .signedness = .signed, + .bits = @bitSizeOf(vector.child), + } })); + return @as(V, @bitCast(rhs)) < @as(V, @splat(0)); + }, + } } inline fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) { switch (@typeInfo(@TypeOf(lhs))) { @@ -69,12 +84,40 @@ noinline fn checkExpected(expected: anytype, actual: @TypeOf(expected)) !void { }) return error.Unexpected; } test checkExpected { + if (checkExpected(nan(f16), nan(f16)) == error.Unexpected) return error.Unexpected; + if (checkExpected(nan(f16), -nan(f16)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f16, 0.0), @as(f16, 0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f16, -0.0), @as(f16, -0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f16, -0.0), @as(f16, 0.0)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f16, 0.0), @as(f16, -0.0)) != error.Unexpected) return error.Unexpected; + if (checkExpected(nan(f32), nan(f32)) == error.Unexpected) return error.Unexpected; if (checkExpected(nan(f32), -nan(f32)) != error.Unexpected) return error.Unexpected; if (checkExpected(@as(f32, 0.0), @as(f32, 0.0)) == error.Unexpected) return error.Unexpected; if (checkExpected(@as(f32, -0.0), @as(f32, -0.0)) == error.Unexpected) return error.Unexpected; if (checkExpected(@as(f32, -0.0), @as(f32, 0.0)) != error.Unexpected) return error.Unexpected; if (checkExpected(@as(f32, 0.0), @as(f32, -0.0)) != error.Unexpected) return error.Unexpected; + + if (checkExpected(nan(f64), nan(f64)) == error.Unexpected) return error.Unexpected; + if (checkExpected(nan(f64), -nan(f64)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f64, 0.0), @as(f64, 0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f64, -0.0), @as(f64, -0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f64, -0.0), @as(f64, 0.0)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f64, 0.0), @as(f64, -0.0)) != error.Unexpected) return error.Unexpected; + + if (checkExpected(nan(f80), nan(f80)) == error.Unexpected) return error.Unexpected; + if (checkExpected(nan(f80), -nan(f80)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f80, 0.0), @as(f80, 0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f80, -0.0), @as(f80, -0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f80, -0.0), @as(f80, 0.0)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f80, 0.0), @as(f80, -0.0)) != error.Unexpected) return error.Unexpected; + + if (checkExpected(nan(f128), nan(f128)) == error.Unexpected) return error.Unexpected; + if (checkExpected(nan(f128), -nan(f128)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f128, 0.0), @as(f128, 0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f128, -0.0), @as(f128, -0.0)) == error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f128, -0.0), @as(f128, 0.0)) != error.Unexpected) return error.Unexpected; + if (checkExpected(@as(f128, 0.0), @as(f128, -0.0)) != error.Unexpected) return error.Unexpected; } fn Unary(comptime op: anytype) type { From b1fa89439ae56001779061c42a08cf9db7906432 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Thu, 23 Jan 2025 05:31:46 -0500 Subject: [PATCH 2/5] x86_64: rewrite float vector `@abs` and equality comparisons --- lib/std/zig/Zir.zig | 8 +- src/Air.zig | 6 + src/InternPool.zig | 36 + src/Sema.zig | 6 + src/Type.zig | 10 +- src/arch/x86_64/CodeGen.zig | 3772 ++++++++++++++++++++++++++++++-- src/arch/x86_64/bits.zig | 1 + src/codegen/c.zig | 196 +- src/codegen/c/Type.zig | 204 +- test/behavior/abs.zig | 2 +- test/behavior/x86_64/build.zig | 1 + test/behavior/x86_64/math.zig | 164 +- test/src/Debugger.zig | 229 +- 13 files changed, 4116 insertions(+), 519 deletions(-) diff --git a/lib/std/zig/Zir.zig b/lib/std/zig/Zir.zig index 7edb99ee03..2ee4053331 100644 --- a/lib/std/zig/Zir.zig +++ b/lib/std/zig/Zir.zig @@ -2128,7 +2128,7 @@ pub const Inst = struct { ref_start_index = static_len, _, - pub const static_len = 70; + pub const static_len = 76; pub fn toRef(i: Index) Inst.Ref { return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i)); @@ -2211,6 +2211,12 @@ pub const Inst = struct { single_const_pointer_to_comptime_int_type, slice_const_u8_type, slice_const_u8_sentinel_0_type, + vector_4_f16_type, + vector_8_f16_type, + vector_4_f32_type, + vector_8_f32_type, + vector_2_f64_type, + vector_4_f64_type, optional_noreturn_type, anyerror_void_error_union_type, adhoc_inferred_error_set_type, diff --git a/src/Air.zig b/src/Air.zig index 3bd658befd..acc1c7f2e2 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -984,6 +984,12 @@ pub const Inst = struct { single_const_pointer_to_comptime_int_type = @intFromEnum(InternPool.Index.single_const_pointer_to_comptime_int_type), slice_const_u8_type = @intFromEnum(InternPool.Index.slice_const_u8_type), slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type), + vector_4_f16_type = @intFromEnum(InternPool.Index.vector_4_f16_type), + vector_8_f16_type = @intFromEnum(InternPool.Index.vector_8_f16_type), + vector_4_f32_type = @intFromEnum(InternPool.Index.vector_4_f32_type), + vector_8_f32_type = @intFromEnum(InternPool.Index.vector_8_f32_type), + vector_2_f64_type = @intFromEnum(InternPool.Index.vector_2_f64_type), + vector_4_f64_type = @intFromEnum(InternPool.Index.vector_4_f64_type), optional_noreturn_type = @intFromEnum(InternPool.Index.optional_noreturn_type), anyerror_void_error_union_type = @intFromEnum(InternPool.Index.anyerror_void_error_union_type), adhoc_inferred_error_set_type = @intFromEnum(InternPool.Index.adhoc_inferred_error_set_type), diff --git a/src/InternPool.zig b/src/InternPool.zig index a92e93705c..04783e2a9c 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -4559,12 +4559,21 @@ pub const Index = enum(u32) { null_type, undefined_type, enum_literal_type, + manyptr_u8_type, manyptr_const_u8_type, manyptr_const_u8_sentinel_0_type, single_const_pointer_to_comptime_int_type, slice_const_u8_type, slice_const_u8_sentinel_0_type, + + vector_4_f16_type, + vector_8_f16_type, + vector_4_f32_type, + vector_8_f32_type, + vector_2_f64_type, + vector_4_f64_type, + optional_noreturn_type, anyerror_void_error_union_type, /// Used for the inferred error set of inline/comptime function calls. @@ -5055,6 +5064,19 @@ pub const static_keys = [_]Key{ }, } }, + // @Vector(4, f16) + .{ .vector_type = .{ .len = 4, .child = .f16_type } }, + // @Vector(8, f16) + .{ .vector_type = .{ .len = 8, .child = .f16_type } }, + // @Vector(4, f32) + .{ .vector_type = .{ .len = 4, .child = .f32_type } }, + // @Vector(8, f32) + .{ .vector_type = .{ .len = 8, .child = .f32_type } }, + // @Vector(2, f64) + .{ .vector_type = .{ .len = 2, .child = .f64_type } }, + // @Vector(4, f64) + .{ .vector_type = .{ .len = 4, .child = .f64_type } }, + // ?noreturn .{ .opt_type = .noreturn_type }, @@ -11681,6 +11703,12 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index { .single_const_pointer_to_comptime_int_type, .slice_const_u8_type, .slice_const_u8_sentinel_0_type, + .vector_4_f16_type, + .vector_8_f16_type, + .vector_4_f32_type, + .vector_8_f32_type, + .vector_2_f64_type, + .vector_4_f64_type, .optional_noreturn_type, .anyerror_void_error_union_type, .adhoc_inferred_error_set_type, @@ -11998,6 +12026,14 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId { .slice_const_u8_sentinel_0_type, => .pointer, + .vector_4_f16_type, + .vector_8_f16_type, + .vector_4_f32_type, + .vector_8_f32_type, + .vector_2_f64_type, + .vector_4_f64_type, + => .vector, + .optional_noreturn_type => .optional, .anyerror_void_error_union_type => .error_union, .empty_tuple_type => .@"struct", diff --git a/src/Sema.zig b/src/Sema.zig index 833b05413f..3aaf8150b5 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -36611,6 +36611,12 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value { .single_const_pointer_to_comptime_int_type, .slice_const_u8_type, .slice_const_u8_sentinel_0_type, + .vector_4_f16_type, + .vector_8_f16_type, + .vector_4_f32_type, + .vector_8_f32_type, + .vector_2_f64_type, + .vector_4_f64_type, .anyerror_void_error_union_type, => null, .void_type => Value.void, diff --git a/src/Type.zig b/src/Type.zig index 0e6ccfa585..4379c9c63f 100644 --- a/src/Type.zig +++ b/src/Type.zig @@ -4174,7 +4174,15 @@ pub const single_const_pointer_to_comptime_int: Type = .{ .ip_index = .single_const_pointer_to_comptime_int_type, }; pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_sentinel_0_type }; -pub const empty_tuple_type: Type = .{ .ip_index = .empty_tuple_type }; + +pub const vector_4_f16: Type = .{ .ip_index = .vector_4_f16_type }; +pub const vector_8_f16: Type = .{ .ip_index = .vector_8_f16_type }; +pub const vector_4_f32: Type = .{ .ip_index = .vector_4_f32_type }; +pub const vector_8_f32: Type = .{ .ip_index = .vector_8_f32_type }; +pub const vector_2_f64: Type = .{ .ip_index = .vector_2_f64_type }; +pub const vector_4_f64: Type = .{ .ip_index = .vector_4_f64_type }; + +pub const empty_tuple: Type = .{ .ip_index = .empty_tuple_type }; pub const generic_poison: Type = .{ .ip_index = .generic_poison_type }; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index b599343730..734b863d7e 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2393,7 +2393,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { - @setEvalBranchQuota(1_800); + @setEvalBranchQuota(2_700); const pt = cg.pt; const zcu = pt.zcu; const ip = &zcu.intern_pool; @@ -2984,6 +2984,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3008,6 +3011,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3032,6 +3038,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3056,6 +3065,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3080,6 +3092,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3104,6 +3119,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3127,6 +3145,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3265,6 +3286,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .each = .{ .once = &.{ @@ -3296,6 +3320,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ @@ -3327,6 +3354,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .each = .{ .once = &.{ @@ -3346,6 +3376,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .each = .{ .once = &.{ @@ -3377,6 +3410,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ @@ -3392,7 +3428,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) }, + .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .ui(0b01111) }, .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ }, } }, }, .{ @@ -3408,6 +3444,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ @@ -3427,6 +3466,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3453,6 +3495,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3477,12 +3522,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, .each = .{ .once = &.{ .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) }, + .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .ui(0b01111) }, .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memiad(.src0y, .tmp0, .add_size, -16), ._ }, .{ ._, .v_pd, .movu, .memiad(.dst0y, .tmp0, .add_size, -16), .tmp2y, ._, ._ }, .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, @@ -3503,12 +3551,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, .each = .{ .once = &.{ .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, - .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .si(0b01111) }, + .{ ._, .v_pd, .cmp, .tmp1y, .tmp1y, .tmp1y, .ui(0b01111) }, .{ .@"0:", .v_pd, .xor, .tmp2y, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._ }, .{ ._, .v_pd, .movu, .memia(.dst0y, .tmp0, .add_size), .tmp2y, ._, ._ }, .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, @@ -3527,6 +3578,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3551,6 +3605,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3576,6 +3633,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3599,6 +3659,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3623,6 +3686,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3647,6 +3713,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3676,6 +3745,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3700,6 +3772,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3725,6 +3800,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3748,6 +3826,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3776,6 +3857,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3800,6 +3884,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3829,6 +3916,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3852,6 +3942,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3880,6 +3973,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3905,6 +4001,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3933,6 +4032,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .clobbers = .{ .eflags = true }, @@ -3957,6 +4059,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -3996,6 +4101,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .each = .{ .once = &.{ @@ -4027,6 +4135,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ @@ -4058,6 +4169,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .each = .{ .once = &.{ @@ -4077,6 +4191,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .ref = .src0 }}, .each = .{ .once = &.{ @@ -4108,6 +4225,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ @@ -4123,7 +4243,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ - .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .si(0b01111) }, + .{ ._, .v_pd, .cmp, .dst0y, .dst0y, .dst0y, .ui(0b01111) }, .{ ._, .v_pd, .xor, .dst0y, .dst0y, .src0y, ._ }, } }, }, .{ @@ -4139,6 +4259,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .sse }}, .each = .{ .once = &.{ @@ -4157,6 +4280,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .each = .{ .once = &.{ @@ -4179,6 +4305,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .umax_mem = .{ .ref = .src0 } } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .each = .{ .once = &.{ @@ -4517,6 +4646,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4541,6 +4673,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4566,6 +4701,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4592,6 +4730,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4650,6 +4791,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4676,6 +4820,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4700,6 +4847,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4722,6 +4872,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4745,6 +4898,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4769,6 +4925,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4894,6 +5053,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -4917,6 +5079,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5041,6 +5206,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5064,6 +5232,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5101,6 +5272,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5156,6 +5330,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5211,6 +5388,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5236,6 +5416,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5258,6 +5441,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5285,6 +5471,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5311,6 +5500,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5339,6 +5531,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5366,6 +5561,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5393,6 +5591,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5419,6 +5620,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5447,6 +5651,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5474,6 +5681,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5504,6 +5714,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5533,6 +5746,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5562,6 +5778,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5592,6 +5811,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5621,6 +5843,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -5650,6 +5875,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5676,6 +5904,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5702,6 +5933,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5728,6 +5962,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5754,6 +5991,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5780,6 +6020,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5806,6 +6049,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5832,6 +6078,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5858,6 +6107,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5887,6 +6139,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5916,6 +6171,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5945,6 +6203,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -5974,6 +6235,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6001,6 +6265,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6029,6 +6296,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6058,6 +6328,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6087,6 +6360,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6116,6 +6392,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6145,6 +6424,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6172,6 +6454,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6200,6 +6485,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6229,6 +6517,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6258,6 +6549,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6287,6 +6581,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6316,6 +6613,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6343,6 +6643,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6371,6 +6674,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6400,6 +6706,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6429,6 +6738,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6458,6 +6770,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6487,6 +6802,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6515,6 +6833,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6544,6 +6865,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6581,6 +6905,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6617,6 +6944,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6653,6 +6983,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6690,6 +7023,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6726,6 +7062,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6762,6 +7101,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6799,6 +7141,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6835,6 +7180,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6871,6 +7219,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6908,6 +7259,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6944,6 +7298,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -6981,8 +7338,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; switch (extra.compareOperator()) { - .eq, .neq => if (cg.typeOf(extra.lhs).scalarType(zcu).isRuntimeFloat()) - break :fallback try cg.airCmpVector(inst), + .eq, .neq => {}, else => break :fallback try cg.airCmpVector(inst), } var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs }); @@ -7004,7 +7360,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_ymm, .to_ymm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7024,7 +7380,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_ymm, .to_ymm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7044,7 +7400,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_ymm, .to_ymm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7064,7 +7420,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_ymm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_ymm, .to_ymm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7084,7 +7440,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_xmm, .to_xmm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7104,7 +7460,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_xmm, .to_xmm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7124,7 +7480,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_xmm, .to_xmm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7144,7 +7500,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .to_xmm }, .commute = .{ 0, 1 } }, .{ .src = .{ .to_xmm, .to_xmm } }, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ + .dst_temps = .{.{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{ .kind = .all, .inverted = switch (cc) { else => unreachable, @@ -7409,6 +7765,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7445,6 +7804,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7487,6 +7849,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7531,6 +7896,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7573,6 +7941,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .sse } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7641,6 +8012,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7683,6 +8057,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7727,6 +8104,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .sse } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7795,6 +8175,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .sse } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7863,6 +8246,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7905,6 +8291,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .sse } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -7949,6 +8338,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .sse } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -8017,6 +8409,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .sse } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -8085,6 +8480,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .mmx } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -8127,6 +8525,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .mmx } }, .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -8199,6 +8600,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .kind = .{ .rc = .mmx } }, .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .kind = .{ .rc = .mmx } }, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -8273,6 +8677,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8302,6 +8709,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8331,6 +8741,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8361,6 +8774,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8390,6 +8806,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8426,6 +8845,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8456,6 +8878,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8486,6 +8911,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8517,6 +8945,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8547,6 +8978,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8585,6 +9019,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8616,6 +9053,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8646,6 +9086,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8677,6 +9120,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8708,6 +9154,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -8744,6 +9193,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -8775,8 +9227,42 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .{ .required_features = .{ .f16c, null, null, null }, .src_constraints = .{ - .{ .scalar_exact_float = .{ .of = .qword, .is = .word } }, - .{ .scalar_exact_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + .{ .scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cvtph2, .dst0x, .src0x, ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp0x, .src1x, ._, ._ }, + .{ ._, .v_ss, .cmp, .dst0x, .dst0x, .tmp0x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .word } }, + .{ .scalar_float = .{ .of = .qword, .is = .word } }, }, .patterns = &.{ .{ .src = .{ .mem, .mem } }, @@ -8791,15 +9277,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ - .kind = .all, - .scalar = .dword, - } } }}, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, .each = .{ .once = &.{ .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ }, - .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .si(switch (cc) { + .{ ._, .v_ps, .cmp, .dst0x, .dst0x, .tmp0x, .ui(switch (cc) { else => unreachable, .e => 0b00000, .ne => 0b00100, @@ -8808,8 +9298,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .{ .required_features = .{ .f16c, null, null, null }, .src_constraints = .{ - .{ .scalar_exact_float = .{ .of = .xword, .is = .word } }, - .{ .scalar_exact_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, + .{ .scalar_float = .{ .of = .xword, .is = .word } }, }, .patterns = &.{ .{ .src = .{ .mem, .mem } }, @@ -8824,24 +9314,2294 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, - .dst_temps = .{.{ .rc_mask = .{ .rc = .sse, .info = .{ - .kind = .all, - .scalar = .dword, - } } }}, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, .each = .{ .once = &.{ .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ }, .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ }, - .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .si(switch (cc) { + .{ ._, .v_ps, .cmp, .dst0y, .dst0y, .tmp0y, .ui(switch (cc) { else => unreachable, .e => 0b00000, .ne => 0b00100, }) }, } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + .{ .scalar_float = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, + .each = .{ .once = &.{ + .{ ._, ._ss, .cmp, .dst0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .scalar_float = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, + .each = .{ .once = &.{ + .{ ._, ._ps, .cmp, .dst0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .dword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .qword }, + } }}, + .each = .{ .once = &.{ + .{ ._, ._sd, .cmp, .dst0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .scalar_float = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref_mask = .{ + .ref = .src0, + .info = .{ .kind = .all, .scalar = .qword }, + } }}, + .each = .{ .once = &.{ + .{ ._, ._pd, .cmp, .dst0x, .src1x, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .scalar_float = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .mut_rc_mask = .{ + .ref = .src0, + .rc = .sse, + .info = .{ .kind = .all, .scalar = .qword }, + } }}, + .each = .{ .once = &.{ + .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + } }, + }, .{ + .required_features = .{ .f16c, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f16, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .f16c, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_f16, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .cvtph2, .tmp2y, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cvtph2, .tmp3y, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .tmp3y, .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_ps, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._c, .in, .tmp1p, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, + .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp1x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .vector_8_f16, .kind = .mem }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .mem(.tmp7x), ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .mem(.tmp7x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._, .add, .tmp0d, .si(1), ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .{ .type = .vector_8_f16, .kind = .mem }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp4d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp1x, .mem(.tmp7x), ._, ._ }, + .{ ._, ._, .movzx, .tmp4d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp7d), .tmp4d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .mem(.tmp7x), ._, ._ }, + .{ ._, ._, .call, .tmp3d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ }, + .{ ._, ._, .@"test", .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp6b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp6d, .tmp5b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp6d, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.none, .tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ }, + .{ ._, .vp_w, .insr, .tmp2x, .tmp3x, .memsi(.src0w, .@"2", .tmp0), .ui(0) }, + .{ ._, .vp_w, .insr, .tmp3x, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0) }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.none, .tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp2x, .memsi(.src0w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, .p_, .xor, .tmp3x, .tmp3x, ._, ._ }, + .{ ._, .p_w, .insr, .tmp3x, .memsi(.src1w, .@"2", .tmp0), .ui(0), ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .vector_8_f16, .kind = .mem }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .mem(.tmp8x), ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .mem(.tmp8x), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._, .lea, .tmp0d, .lead(.none, .tmp0, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f16, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f16, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqhf2", + .ne => "__nehf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + .{ .type = .vector_8_f16, .kind = .mem }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp5d, .memsi(.src0w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .mem(.tmp8x), ._, ._ }, + .{ ._, ._, .movzx, .tmp5d, .memsi(.src1w, .@"2", .tmp0), ._, ._ }, + .{ ._, ._, .mov, .mem(.tmp8d), .tmp5d, ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .mem(.tmp8x), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp0d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7q, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .tmp1q, .tmp7q, ._, ._ }, + .{ ._, ._c, .in, .tmp0d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp5, -8), .tmp1q, ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"1:", ._, .cmp, .tmp0d, .sa(.src0, .add_len), ._, ._ }, + .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp5d, .tmp0d, ._, ._ }, + .{ ._, ._r, .sh, .tmp5d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp5), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_ps, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_ps, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_ps, .movmsk, .tmp3d, .tmp2y, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, ._ps, .cmp, .tmp3x, .memiad(.src1x, .tmp0, .add_size, 16), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, .p_w, .ackssd, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, 16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, 16), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memiad(.src0x, .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memiad(.src1x, .tmp0, .add_size, -16), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp4d, .tmp2x, ._, ._ }, + .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.none, .tmp1, 1), ._, ._ }, + .{ .@"1:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .cmp, .tmp2x, .memia(.src1x, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, ._ps, .movmsk, .tmp3d, .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp3b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .zword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp3y, .memiad(.src0y, .tmp0, .add_size, 32), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp2y, .tmp2y, .memia(.src1y, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memiad(.src1y, .tmp0, .add_size, 32), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp2y, ._, ._ }, + .{ ._, .v_pd, .movmsk, .tmp5d, .tmp3y, ._, ._ }, + .{ ._, ._l, .sh, .tmp5b, .ui(4), ._, ._ }, + .{ ._, ._, .@"or", .tmp4b, .tmp5b, ._, ._ }, + .{ ._, ._, .mov, .lea(.byte, .tmp1), .tmp4b, ._, ._ }, + .{ ._, ._c, .in, .tmp1q, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_pd, .mova, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_pd, .cmp, .tmp3y, .tmp3y, .memia(.src1y, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }) }, + .{ ._, .v_pd, .movmsk, .tmp4d, .tmp3y, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 4), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .cmp, .tmp3x, .memia(.src1x, .tmp0, .add_size), .ui(switch (cc) { + else => unreachable, + .e => 0b00000, + .ne => 0b00100, + }), ._ }, + .{ ._, ._pd, .movmsk, .tmp4d, .tmp3x, ._, ._ }, + .{ ._, ._l, .ro, .tmp4b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp4b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 2), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp4, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp4d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp4d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp4), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u8, .kind = .{ .reg = .ah } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0q, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6b, .si(0b0_1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp6b, .si(0b0_1_000_100), ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_p, .ucomi, .tmp5t, .tmp6t, ._, ._ }, + .{ ._, .f_p, .st, .tmp6t, ._, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, switch (cc) { + else => unreachable, + .e => ._np, + .ne => ._p, + }, .set, .tmp4b, ._, ._, ._ }, + .{ ._, ._, switch (cc) { + else => unreachable, + .e => .@"and", + .ne => .@"or", + }, .tmp3b, .tmp4b, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .x87, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .reg = .rcx } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f80, .kind = .{ .reg = .st6 } }, + .{ .type = .f80, .kind = .{ .reg = .st7 } }, + .{ .type = .u8, .kind = .{ .reg = .ah } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_, .ld, .memia(.src0t, .tmp0, .add_size), ._, ._, ._ }, + .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, + .{ ._, .fn_sw, .st, .tmp6w, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp6b, .si(0b0_1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp6b, .si(0b0_1_000_100), ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, + .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0b, .tmp3, -1), .tmp2b, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp3d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memi(.dst0b, .tmp3), .tmp2b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, .slow_incdec, null, null }, + .dst_constraints = .{.{ .bool_vec = .dword }}, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u32, .kind = .{ .reg = .edx } }, + .unused, + }, + .dst_temps = .{.{ .rc = .general_purpose }}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp2x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp3x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp4d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ }, + .{ ._, ._, .@"test", .tmp5d, .tmp5d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp7b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp7d, .tmp6b, ._, ._ }, + .{ ._, ._, .@"or", .dst0d, .tmp7d, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .avx, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse2, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, .slow_incdec, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, 1), ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .sse, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .call_frame = .{ .alignment = .@"16" }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f128, .kind = .{ .reg = .xmm0 } }, + .{ .type = .f128, .kind = .{ .reg = .xmm1 } }, + .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { + else => unreachable, + .e => "__eqtf2", + .ne => "__netf2", + } } } }, + .{ .type = .i32, .kind = .{ .reg = .eax } }, + .{ .type = .u8, .kind = .{ .reg = .cl } }, + .{ .type = .u64, .kind = .{ .reg = .rdx } }, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .call, .tmp5d, ._, ._, ._ }, + .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ }, + .{ ._, ._, .@"test", .tmp6d, .tmp6d, ._, ._ }, + .{ ._, .fromCondition(cc), .set, .tmp8b, ._, ._, ._ }, + .{ ._, ._, .mov, .tmp7d, .tmp1d, ._, ._ }, + .{ ._, ._l, .sh, .tmp8q, .tmp7b, ._, ._ }, + .{ ._, ._, .@"or", .tmp2q, .tmp8q, ._, ._ }, + .{ ._, ._c, .in, .tmp1d, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .ui(3), ._, ._ }, + .{ ._, ._, .mov, .memid(.dst0q, .tmp6, -8), .tmp2q, ._, ._ }, + .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, + .{ .@"1:", ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp1d, .si(0b111111), ._, ._ }, + .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp6d, .tmp1d, ._, ._ }, + .{ ._, ._r, .sh, .tmp6d, .si(9), ._, ._ }, + .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp6), .tmp2q, ._, ._ }, + } }, } }, }) catch |err| switch (err) { - error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ + error.SelectFailed => return cg.fail("failed to select {s} {s} {} {} {}", .{ @tagName(air_tag), + @tagName(extra.compareOperator()), cg.typeOf(extra.lhs).fmt(pt), ops[0].tracking(cg), ops[1].tracking(cg), @@ -8854,9 +11614,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try res[0].finish(inst, &.{ extra.lhs, extra.rhs }, &ops, cg); }, - .abs => |air_tag| if (use_old) try cg.airAbs(inst) else fallback: { + .abs => |air_tag| if (use_old) try cg.airAbs(inst) else { const ty_op = air_datas[@intFromEnum(inst)].ty_op; - if (ty_op.ty.toType().isVector(zcu) and ty_op.ty.toType().childType(zcu).isRuntimeFloat()) break :fallback try cg.airAbs(inst); var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); var res: [1]Temp = undefined; cg.select(&res, &.{ty_op.ty.toType()}, &ops, comptime &.{ .{ @@ -9037,6 +11796,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9061,7 +11823,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_mm, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .mmx } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }}, .each = .{ .once = &.{ .{ ._, .p_b, .abs, .dst0q, .src0q, ._, ._ }, } }, @@ -9072,7 +11834,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_mm, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .mmx } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }}, .each = .{ .once = &.{ .{ ._, .p_w, .abs, .dst0q, .src0q, ._, ._ }, } }, @@ -9083,7 +11845,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_mm, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .mmx } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .mmx } }}, .each = .{ .once = &.{ .{ ._, .p_d, .abs, .dst0q, .src0q, ._, ._ }, } }, @@ -9094,7 +11856,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .p_b, .abs, .dst0x, .src0x, ._, ._ }, } }, @@ -9105,7 +11867,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .p_w, .abs, .dst0x, .src0x, ._, ._ }, } }, @@ -9116,7 +11878,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .p_d, .abs, .dst0x, .src0x, ._, ._ }, } }, @@ -9127,7 +11889,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .vp_b, .abs, .dst0x, .src0x, ._, ._ }, } }, @@ -9138,7 +11900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .vp_w, .abs, .dst0x, .src0x, ._, ._ }, } }, @@ -9149,7 +11911,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .vp_d, .abs, .dst0x, .src0x, ._, ._ }, } }, @@ -9160,7 +11922,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .vp_b, .abs, .dst0y, .src0y, ._, ._ }, } }, @@ -9171,7 +11933,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .vp_w, .abs, .dst0y, .src0y, ._, ._ }, } }, @@ -9182,7 +11944,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_sse, .none } }, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, .vp_d, .abs, .dst0y, .src0y, ._, ._ }, } }, @@ -9199,6 +11961,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9222,6 +11987,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9245,6 +12013,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9268,6 +12039,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9291,6 +12065,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9314,6 +12091,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9337,6 +12117,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9360,6 +12143,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9383,6 +12169,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9406,6 +12195,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9429,6 +12221,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9452,6 +12247,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9475,6 +12273,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9501,6 +12302,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9527,6 +12331,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9552,6 +12359,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9578,6 +12388,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9602,6 +12415,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9628,6 +12444,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9652,6 +12471,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9678,6 +12500,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9703,6 +12528,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9729,6 +12557,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.mem}, .clobbers = .{ .eflags = true }, @@ -9750,15 +12581,40 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._, .cmp, .tmp0d, .sa(.none, .add_src0_unaligned_size), ._, ._ }, .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ }, + } }, }, .{ .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .dword } }, .any }, .patterns = &.{ .{ .src = .{ .to_mut_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -9769,15 +12625,62 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, .{ ._, ._ps, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ }, } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_sse, .none } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ }, + } }, }, .{ .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .qword } }, .any }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .qword } }, .any }, .patterns = &.{ .{ .src = .{ .to_mut_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -9788,104 +12691,31 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, .{ ._, ._pd, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ }, } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_mut_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{.{ .ref = .src0 }}, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, ._ps, .@"and", .dst0x, .lea(.xword, .tmp0), ._, ._ }, - } }, }, .{ .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .dword } }, .any }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .qword } }, .any }, .patterns = &.{ .{ .src = .{ .to_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, .unused, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_pd, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .yword, .is = .dword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .v_ps, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .yword, .is = .qword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ }, } }, }, .{ .required_features = .{ .x87, null, null, null }, - .src_constraints = .{ .{ .scalar_exact_float = .{ .of = .xword, .is = .tbyte } }, .any }, + .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any }, .patterns = &.{ .{ .src = .{ .mem, .none } }, .{ .src = .{ .to_x87, .none } }, @@ -9897,60 +12727,50 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .x87 } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .x87 } }}, .each = .{ .once = &.{ .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, .{ ._, .f_, .abs, ._, ._, ._, ._ }, .{ ._, .f_p, .st, .dst0t, ._, ._, ._ }, } }, - }, .{ - .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, - .patterns = &.{ - .{ .src = .{ .to_sse, .none } }, - }, - .extra_temps = .{ - .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, - .unused, - .unused, - .unused, - .unused, - }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, - .each = .{ .once = &.{ - .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, - .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ }, - } }, }, .{ .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any }, .patterns = &.{ .{ .src = .{ .to_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, .unused, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, .{ ._, .vp_, .@"and", .dst0x, .src0x, .lea(.xword, .tmp0), ._ }, } }, }, .{ .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any }, .patterns = &.{ .{ .src = .{ .to_mut_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -9963,13 +12783,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .scalar_any_float = .xword }, .any }, .patterns = &.{ .{ .src = .{ .to_mut_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -9982,42 +12805,268 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } }, }, .{ .required_features = .{ .avx2, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .scalar_any_float = .yword }, .any }, .patterns = &.{ .{ .src = .{ .to_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, .unused, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, .{ ._, .vp_, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ }, } }, }, .{ .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .{ .scalar_float = .{ .of = .yword, .is = .xword } }, .any }, + .src_constraints = .{ .{ .scalar_any_float = .yword }, .any }, .patterns = &.{ .{ .src = .{ .to_sse, .none } }, }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, - .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize = true } } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, .unused, }, - .dst_temps = .{.{ .mut_reg = .{ .ref = .src0, .rc = .sse } }}, + .dst_temps = .{.{ .mut_rc = .{ .ref = .src0, .rc = .sse } }}, .each = .{ .once = &.{ .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, .{ ._, .v_pd, .@"and", .dst0y, .src0y, .lea(.yword, .tmp0), ._ }, } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_ps, .mova, .tmp2y, .lea(.yword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_ps, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lea(.yword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._pd, .mova, .tmp2x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._pd, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._pd, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .lea(.yword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .vp_, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .yword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .yword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_pd, .mova, .tmp2y, .lea(.yword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_pd, .@"and", .tmp3y, .tmp2y, .memia(.src0y, .tmp0, .add_size), ._ }, + .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, null, null, null }, + .src_constraints = .{ .{ .multiple_scalar_any_float = .xword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smax_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .kind = .{ .rc = .sse } }, + .{ .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._ps, .mova, .tmp2x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._ps, .mova, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, ._ps, .@"and", .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, } }) catch |err| switch (err) { error.SelectFailed => return cg.fail("failed to select {s} {} {}", .{ @tagName(air_tag), @@ -10194,6 +13243,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .rc = .general_purpose }}, .clobbers = .{ .eflags = true }, @@ -10328,6 +13380,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10353,6 +13408,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10389,6 +13447,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10414,6 +13475,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10436,6 +13500,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10459,6 +13526,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10491,6 +13561,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10520,6 +13593,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10552,6 +13628,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10581,6 +13660,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10610,6 +13692,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10639,6 +13724,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10671,6 +13759,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10702,6 +13793,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -10728,6 +13822,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10747,7 +13844,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .patterns = &.{ .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, }, - .call_frame = .{ .size = 0, .alignment = .@"16" }, + .call_frame = .{ .alignment = .@"16" }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { else => unreachable, @@ -10759,11 +13856,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, } }, }, .{ @@ -10781,6 +13881,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10806,6 +13909,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10822,7 +13928,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .patterns = &.{ .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, }, - .call_frame = .{ .size = 0, .alignment = .@"16" }, + .call_frame = .{ .alignment = .@"16" }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { else => unreachable, @@ -10834,11 +13940,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, } }, }, .{ @@ -10856,6 +13965,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10881,6 +13993,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10897,7 +14012,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .patterns = &.{ .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, }, - .call_frame = .{ .size = 0, .alignment = .@"16" }, + .call_frame = .{ .alignment = .@"16" }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { else => unreachable, @@ -10909,11 +14024,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, } }, }, .{ @@ -10929,6 +14047,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10957,6 +14078,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = switch (cc) { else => unreachable, @@ -10977,7 +14101,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .extra_temps = .{ .{ .type = .f80, .kind = .{ .reg = .st6 } }, .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u8, .kind = .{ .reg = .ah } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -11004,10 +14131,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .extra_temps = .{ .{ .type = .f80, .kind = .{ .reg = .st6 } }, .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, .{ .type = .u8, .kind = .{ .reg = .ah } }, .unused, .unused, + .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -11016,8 +14146,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, .{ ._, .f_pp, .ucom, ._, ._, ._, ._ }, .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b0_1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .si(0b0_1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp2b, .si(0b0_1_000_100), ._, ._ }, } }, }, .{ .required_features = .{ .x87, null, null, null }, @@ -11028,7 +14158,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .extra_temps = .{ .{ .type = .f80, .kind = .{ .reg = .st6 } }, .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u8, .kind = .{ .reg = .ah } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -11057,7 +14190,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .extra_temps = .{ .{ .type = .f80, .kind = .{ .reg = .st6 } }, .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u16, .kind = .{ .reg = .ah } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -11085,10 +14221,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .extra_temps = .{ .{ .type = .f80, .kind = .{ .reg = .st6 } }, .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, .{ .type = .u8, .kind = .{ .reg = .ah } }, .unused, .unused, + .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true }, @@ -11096,8 +14235,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, .f_, .ld, .src0t, ._, ._, ._ }, .{ ._, .f_p, .ucom, .src1t, ._, ._, ._ }, .{ ._, .fn_sw, .st, .tmp2w, ._, ._, ._ }, - .{ ._, ._, .xor, .tmp3b, .si(0b0_1_000_000), ._, ._ }, - .{ ._, ._, .@"test", .tmp3b, .si(0b0_1_000_100), ._, ._ }, + .{ ._, ._, .xor, .tmp2b, .si(0b0_1_000_000), ._, ._ }, + .{ ._, ._, .@"test", .tmp2b, .si(0b0_1_000_100), ._, ._ }, } }, }, .{ .required_features = .{ .x87, null, null, null }, @@ -11110,7 +14249,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .extra_temps = .{ .{ .type = .f80, .kind = .{ .reg = .st6 } }, .{ .type = .f80, .kind = .{ .reg = .st7 } }, - .{ .type = .u16, .kind = .{ .reg = .ax } }, + .{ .type = .u16, .kind = .{ .reg = .ah } }, + .unused, + .unused, + .unused, .unused, .unused, .unused, @@ -11133,7 +14275,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .patterns = &.{ .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 } } }, }, - .call_frame = .{ .size = 0, .alignment = .@"16" }, + .call_frame = .{ .alignment = .@"16" }, .extra_temps = .{ .{ .type = .usize, .kind = .{ .symbol = &.{ .name = switch (cc) { else => unreachable, @@ -11145,11 +14287,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .unused, .unused, .unused, + .unused, + .unused, + .unused, }, .dst_temps = .{.{ .cc = cc }}, .clobbers = .{ .eflags = true, .caller_preserved = .ccc }, .each = .{ .once = &.{ - .{ ._, ._, .call, .tmp0p, ._, ._, ._ }, + .{ ._, ._, .call, .tmp0d, ._, ._, ._ }, .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ }, } }, } }, @@ -11753,7 +14898,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .err_return_trace => { const ert: Temp = .{ .index = err_ret_trace_index }; - try ert.moveTo(inst, cg); + try ert.finish(inst, &.{}, &.{}, cg); }, .set_err_return_trace => { const un_op = air_datas[@intFromEnum(inst)].un_op; @@ -11788,7 +14933,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { var ert: Temp = .{ .index = err_ret_trace_index }; var res = try ert.load(.usize, .{ .disp = @intCast(agg_ty.structFieldOffset(ty_pl.payload, zcu)) }, cg); try ert.die(cg); - try res.moveTo(inst, cg); + try res.finish(inst, &.{}, &.{}, cg); }, .vector_store_elem => return cg.fail("TODO implement vector_store_elem", .{}), @@ -23113,7 +26258,11 @@ fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index .resurrect = true, .close_scope = true, }); - for (block_data.value.relocs.items) |reloc| self.performReloc(reloc); + const block_relocs_last_index = block_data.value.relocs.items.len - 1; + for (if (block_data.value.relocs.items[block_relocs_last_index] == self.mir_instructions.len - 1) block_relocs: { + _ = self.mir_instructions.pop(); + break :block_relocs block_data.value.relocs.items[0..block_relocs_last_index]; + } else block_data.value.relocs.items) |block_reloc| self.performReloc(block_reloc); } if (std.debug.runtime_safety) assert(self.inst_tracking.getIndex(inst).? == inst_tracking_i); @@ -24623,13 +27772,15 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) 64 => switch (ty.vectorLen(zcu)) { 1...2 => return .{ .move = if (self.hasFeature(.avx)) .{ .v_pd, if (aligned) .mova else .movu } + else if (self.hasFeature(.sse2)) + .{ ._pd, if (aligned) .mova else .movu } else - .{ ._pd, if (aligned) .mova else .movu } }, + .{ ._ps, if (aligned) .mova else .movu } }, 3...4 => if (self.hasFeature(.avx)) return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } }, else => {}, }, - 128 => switch (ty.vectorLen(zcu)) { + 80, 128 => switch (ty.vectorLen(zcu)) { 1 => return .{ .move = if (self.hasFeature(.avx)) .{ if (aligned) .v_dqa else .v_dqu, .mov } else if (self.hasFeature(.sse2)) @@ -24958,7 +28109,7 @@ fn genSetReg( }, .mmx => unreachable, .sse => switch (src_reg.class()) { - .general_purpose => try self.asmRegisterRegister( + .general_purpose => if (self.hasFeature(.sse2)) try self.asmRegisterRegister( switch (abi_size) { 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, @@ -24966,7 +28117,26 @@ fn genSetReg( }, dst_reg.to128(), registerAlias(src_reg, @max(abi_size, 4)), - ), + ) else { + const frame_size = std.math.ceilPowerOfTwoAssert(u32, @max(abi_size, 4)); + const frame_index = try self.allocFrameIndex(.init(.{ + .size = frame_size, + .alignment = .fromNonzeroByteUnits(frame_size), + })); + try self.asmMemoryRegister(.{ ._, .mov }, .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, + }, registerAlias(src_reg, abi_size)); + try self.asmRegisterMemory(switch (frame_size) { + 4 => .{ ._ss, .mov }, + 8 => .{ ._ps, .movl }, + 16 => .{ ._ps, .mov }, + else => unreachable, + }, dst_reg.to128(), .{ + .base = .{ .frame = frame_index }, + .mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, + }); + }, .segment => try self.genSetReg( dst_reg, ty, @@ -24977,7 +28147,12 @@ fn genSetReg( .sse => try self.asmRegisterRegister( @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) { else => switch (abi_size) { - 1...16 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else .{ ._dqa, .mov }, + 1...16 => if (self.hasFeature(.avx)) + .{ .v_dqa, .mov } + else if (self.hasFeature(.sse2)) + .{ ._dqa, .mov } + else + .{ ._ps, .mova }, 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, else => null, }, @@ -24985,13 +28160,20 @@ fn genSetReg( 16, 128 => switch (abi_size) { 2...16 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } + else if (self.hasFeature(.sse2)) + .{ ._dqa, .mov } else - .{ ._dqa, .mov }, + .{ ._ps, .mova }, 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, else => null, }, 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova }, - 64 => if (self.hasFeature(.avx)) .{ .v_pd, .mova } else .{ ._pd, .mova }, + 64 => if (self.hasFeature(.avx)) + .{ .v_pd, .mova } + else if (self.hasFeature(.sse2)) + .{ ._pd, .mova } + else + .{ ._ps, .mova }, 80 => null, else => unreachable, }, @@ -25688,15 +28870,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { const src_lock = if (src_mcv.getReg()) |src_reg| self.register_manager.lockReg(src_reg) else null; defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - const dst_mcv = if ((if (src_mcv.getReg()) |src_reg| src_reg.class() == .general_purpose else true) and + const dst_mcv = if (src_mcv != .register_mask and + (if (src_mcv.getReg()) |src_reg| src_reg.class() == .general_purpose else true) and dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and dst_ty.abiAlignment(zcu).order(src_ty.abiAlignment(zcu)).compare(.lte) and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { - .lt => dst_ty, - .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty, - .gt => src_ty, + try self.genCopy(switch (src_mcv) { + else => switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { + .lt => dst_ty, + .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty, + .gt => src_ty, + }, + .register_mask => src_ty, }, dst_mcv, src_mcv, .{}); break :dst dst_mcv; }; @@ -31173,7 +34359,7 @@ const Select = struct { dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any), src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any), patterns: []const Select.Pattern, - call_frame: packed struct(u16) { size: u10, alignment: InternPool.Alignment } = .{ .size = 0, .alignment = .none }, + call_frame: packed struct(u16) { size: u10 = 0, alignment: InternPool.Alignment } = .{ .size = 0, .alignment = .none }, extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused), dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused), clobbers: packed struct { @@ -31202,14 +34388,16 @@ const Select = struct { multiple_size: Memory.Size, int: Memory.Size, scalar_int_is: Memory.Size, - scalar_int: struct { of: Memory.Size, is: Memory.Size }, + scalar_int: OfIsSizes, scalar_signed_int: Memory.Size, scalar_unsigned_int: Memory.Size, - scalar_remainder_int: struct { of: Memory.Size, is: Memory.Size }, + multiple_scalar_int: OfIsSizes, + scalar_remainder_int: OfIsSizes, float: Memory.Size, - scalar_float: struct { of: Memory.Size, is: Memory.Size }, - scalar_exact_float: struct { of: Memory.Size, is: Memory.Size }, - multiple_scalar_int: struct { of: Memory.Size, is: Memory.Size }, + scalar_any_float: Memory.Size, + scalar_float: OfIsSizes, + multiple_scalar_any_float: Memory.Size, + multiple_scalar_float: OfIsSizes, exact_int: u16, exact_signed_int: u16, exact_unsigned_int: u16, @@ -31218,16 +34406,18 @@ const Select = struct { po2_int: Memory.Size, signed_po2_int: Memory.Size, unsigned_po2_or_exact_int: Memory.Size, - remainder_int: struct { of: Memory.Size, is: Memory.Size }, - exact_remainder_int: struct { of: Memory.Size, is: Memory.Size }, - signed_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size }, - unsigned_or_exact_remainder_int: struct { of: Memory.Size, is: Memory.Size }, + remainder_int: OfIsSizes, + exact_remainder_int: OfIsSizes, + signed_or_exact_remainder_int: OfIsSizes, + unsigned_or_exact_remainder_int: OfIsSizes, signed_int: Memory.Size, unsigned_int: Memory.Size, elem_size_is: u8, po2_elem_size, elem_int: Memory.Size, + const OfIsSizes = struct { of: Memory.Size, is: Memory.Size }; + fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool { const zcu = cg.pt.zcu; return switch (constraint) { @@ -31258,7 +34448,7 @@ const Select = struct { size.bitSize(cg.target) >= int_info.bits else false, - .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, .scalar_signed_int => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { .signed => size.bitSize(cg.target) >= int_info.bits, @@ -31275,9 +34465,13 @@ const Select = struct { else false, .float => |size| if (cg.floatBits(ty)) |float_bits| size.bitSize(cg.target) == float_bits else false, - .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and - if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) >= float_bits else false, - .scalar_exact_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + .scalar_any_float => |size| @divExact(size.bitSize(cg.target), 8) >= ty.abiSize(zcu) and + cg.floatBits(ty.scalarType(zcu)) != null, + .scalar_float => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, + .multiple_scalar_any_float => |size| ty.abiSize(zcu) % @divExact(size.bitSize(cg.target), 8) == 0 and + cg.floatBits(ty.scalarType(zcu)) != null, + .multiple_scalar_float => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and if (cg.floatBits(ty.scalarType(zcu))) |float_bits| of_is.is.bitSize(cg.target) == float_bits else false, .exact_int => |bit_size| if (cg.intInfo(ty)) |int_info| bit_size == int_info.bits else false, .exact_signed_int => |bit_size| if (cg.intInfo(ty)) |int_info| switch (int_info.signedness) { @@ -31505,48 +34699,81 @@ const Select = struct { unused, any, cc: Condition, + ref: Select.Operand.Ref, reg: Register, rc: Register.Class, + mut_rc: struct { ref: Select.Operand.Ref, rc: Register.Class }, + ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, rc_mask: struct { rc: Register.Class, info: MaskInfo }, + mut_rc_mask: struct { ref: Select.Operand.Ref, rc: Register.Class, info: MaskInfo }, mem, smin_mem: ConstInfo, smax_mem: ConstInfo, umin_mem: ConstInfo, umax_mem: ConstInfo, - ref: Select.Operand.Ref, - ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, - mut_reg: struct { ref: Select.Operand.Ref, rc: Register.Class }, symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 }, - const ConstInfo = struct { ref: Select.Operand.Ref, vectorize: bool = false }; + const ConstInfo = struct { ref: Select.Operand.Ref, vectorize_to: Memory.Size = .none }; fn finish(kind: Kind, temp: Temp, s: *const Select) void { switch (kind) { else => {}, - inline .rc_mask, .ref_mask => |mask| temp.asMask(mask.info, s.cg), + inline .rc_mask, .mut_rc_mask, .ref_mask => |mask| temp.asMask(mask.info, s.cg), } } }; - fn create(spec: TempSpec, s: *Select) !?Temp { + fn pass(spec: TempSpec) u2 { + return switch (spec.kind) { + .unused => 0, + .reg => 1, + else => 2, + }; + } + + fn create(spec: TempSpec, s: *Select) !Temp { const cg = s.cg; return switch (spec.kind) { - .unused => null, + .unused => unreachable, .any => try cg.tempAlloc(spec.type), .cc => |cc| try cg.tempInit(spec.type, .{ .eflags = cc }), + .ref => |ref| ref.deref(s), .reg => |reg| try cg.tempInit(spec.type, .{ .register = reg }), .rc => |rc| try cg.tempAllocReg(spec.type, regSetForRegClass(rc)), + .mut_rc => |ref_rc| { + const temp = ref_rc.ref.deref(s); + if (temp.isMut(cg)) switch (temp.tracking(cg).short) { + .register => |reg| if (reg.class() == ref_rc.rc) return temp, + .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.class() == ref_rc.rc) return temp, + else => {}, + }; + return try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc)); + }, + .ref_mask => |ref_mask| ref_mask.ref.deref(s), .rc_mask => |rc_mask| try cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)), + .mut_rc_mask => |ref_rc_mask| { + const temp = ref_rc_mask.ref.deref(s); + if (temp.isMut(cg)) switch (temp.tracking(cg).short) { + .register => |reg| if (reg.class() == ref_rc_mask.rc) return temp, + .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.class() == ref_rc_mask.rc) return temp, + else => {}, + }; + return try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc_mask.rc)); + }, .mem => try cg.tempAllocMem(spec.type), .smin_mem, .smax_mem, .umin_mem, .umax_mem => |const_info| { const pt = cg.pt; const zcu = pt.zcu; const ip = &zcu.intern_pool; - const ty = const_info.ref.deref(s).typeOf(s.cg); - const vector_len: ?u32, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) { - else => .{ if (const_info.vectorize) 1 else null, ty }, + const ty = const_info.ref.deref(s).typeOf(cg); + const vector_len, const scalar_ty: Type = switch (ip.indexToKey(ty.toIntern())) { + else => .{ null, ty }, .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) }, }; + const res_vector_len: ?u32 = if (const_info.vectorize_to != .none) + @intCast(@divExact(@divExact(const_info.vectorize_to.bitSize(cg.target), 8), scalar_ty.abiSize(pt.zcu))) + else + vector_len; const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) { .bool_type => .{ scalar_ty, @@ -31591,7 +34818,7 @@ const Select = struct { break :res_scalar .{ scalar_int_ty, try pt.intValue_big(scalar_int_ty, big_int.toConst()) }; }, }; - const res_val: Value = if (vector_len) |len| .fromInterned(try pt.intern(.{ .aggregate = .{ + const res_val: Value = if (res_vector_len) |len| .fromInterned(try pt.intern(.{ .aggregate = .{ .ty = (try pt.vectorType(.{ .len = len, .child = res_scalar_ty.toIntern(), @@ -31600,17 +34827,6 @@ const Select = struct { } })) else res_scalar_val; return try cg.tempFromValue(res_val); }, - .ref => |ref| ref.deref(s), - .ref_mask => |ref_mask| ref_mask.ref.deref(s), - .mut_reg => |ref_rc| { - const temp = ref_rc.ref.deref(s); - if (temp.isMut(cg)) switch (temp.tracking(cg).short) { - .register => |reg| if (reg.class() == ref_rc.rc) return temp, - .register_offset => |reg_off| if (reg_off.off == 0 and reg_off.reg.class() == ref_rc.rc) return temp, - else => {}, - }; - return try cg.tempAllocReg(spec.type, regSetForRegClass(ref_rc.rc)); - }, .symbol => |symbol| if (cg.bin_file.cast(.elf)) |elf_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ .sym_index = try elf_file.getGlobalSymbol(symbol.name, symbol.lib), } }) else if (cg.bin_file.cast(.macho)) |macho_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ @@ -31707,6 +34923,9 @@ const Select = struct { tmp3, tmp4, tmp5, + tmp6, + tmp7, + tmp8, dst0, src0, src1, @@ -31778,6 +34997,36 @@ const Select = struct { const tmp5x: Sized = .{ .ref = .tmp5, .size = .xword }; const tmp5y: Sized = .{ .ref = .tmp5, .size = .yword }; + const tmp6: Sized = .{ .ref = .tmp6, .size = .none }; + const tmp6b: Sized = .{ .ref = .tmp6, .size = .byte }; + const tmp6w: Sized = .{ .ref = .tmp6, .size = .word }; + const tmp6d: Sized = .{ .ref = .tmp6, .size = .dword }; + const tmp6p: Sized = .{ .ref = .tmp6, .size = .ptr }; + const tmp6q: Sized = .{ .ref = .tmp6, .size = .qword }; + const tmp6t: Sized = .{ .ref = .tmp6, .size = .tbyte }; + const tmp6x: Sized = .{ .ref = .tmp6, .size = .xword }; + const tmp6y: Sized = .{ .ref = .tmp6, .size = .yword }; + + const tmp7: Sized = .{ .ref = .tmp7, .size = .none }; + const tmp7b: Sized = .{ .ref = .tmp7, .size = .byte }; + const tmp7w: Sized = .{ .ref = .tmp7, .size = .word }; + const tmp7d: Sized = .{ .ref = .tmp7, .size = .dword }; + const tmp7p: Sized = .{ .ref = .tmp7, .size = .ptr }; + const tmp7q: Sized = .{ .ref = .tmp7, .size = .qword }; + const tmp7t: Sized = .{ .ref = .tmp7, .size = .tbyte }; + const tmp7x: Sized = .{ .ref = .tmp7, .size = .xword }; + const tmp7y: Sized = .{ .ref = .tmp7, .size = .yword }; + + const tmp8: Sized = .{ .ref = .tmp8, .size = .none }; + const tmp8b: Sized = .{ .ref = .tmp8, .size = .byte }; + const tmp8w: Sized = .{ .ref = .tmp8, .size = .word }; + const tmp8d: Sized = .{ .ref = .tmp8, .size = .dword }; + const tmp8p: Sized = .{ .ref = .tmp8, .size = .ptr }; + const tmp8q: Sized = .{ .ref = .tmp8, .size = .qword }; + const tmp8t: Sized = .{ .ref = .tmp8, .size = .tbyte }; + const tmp8x: Sized = .{ .ref = .tmp8, .size = .xword }; + const tmp8y: Sized = .{ .ref = .tmp8, .size = .yword }; + const dst0: Sized = .{ .ref = .dst0, .size = .none }; const dst0b: Sized = .{ .ref = .dst0, .size = .byte }; const dst0w: Sized = .{ .ref = .dst0, .size = .word }; @@ -31875,6 +35124,33 @@ const Select = struct { const tmp5x: Select.Operand = .{ .tag = .ref, .base = .tmp5x }; const tmp5y: Select.Operand = .{ .tag = .ref, .base = .tmp5y }; + const tmp6b: Select.Operand = .{ .tag = .ref, .base = .tmp6b }; + const tmp6w: Select.Operand = .{ .tag = .ref, .base = .tmp6w }; + const tmp6d: Select.Operand = .{ .tag = .ref, .base = .tmp6d }; + const tmp6p: Select.Operand = .{ .tag = .ref, .base = .tmp6p }; + const tmp6q: Select.Operand = .{ .tag = .ref, .base = .tmp6q }; + const tmp6t: Select.Operand = .{ .tag = .ref, .base = .tmp6t }; + const tmp6x: Select.Operand = .{ .tag = .ref, .base = .tmp6x }; + const tmp6y: Select.Operand = .{ .tag = .ref, .base = .tmp6y }; + + const tmp7b: Select.Operand = .{ .tag = .ref, .base = .tmp7b }; + const tmp7w: Select.Operand = .{ .tag = .ref, .base = .tmp7w }; + const tmp7d: Select.Operand = .{ .tag = .ref, .base = .tmp7d }; + const tmp7p: Select.Operand = .{ .tag = .ref, .base = .tmp7p }; + const tmp7q: Select.Operand = .{ .tag = .ref, .base = .tmp7q }; + const tmp7t: Select.Operand = .{ .tag = .ref, .base = .tmp7t }; + const tmp7x: Select.Operand = .{ .tag = .ref, .base = .tmp7x }; + const tmp7y: Select.Operand = .{ .tag = .ref, .base = .tmp7y }; + + const tmp8b: Select.Operand = .{ .tag = .ref, .base = .tmp8b }; + const tmp8w: Select.Operand = .{ .tag = .ref, .base = .tmp8w }; + const tmp8d: Select.Operand = .{ .tag = .ref, .base = .tmp8d }; + const tmp8p: Select.Operand = .{ .tag = .ref, .base = .tmp8p }; + const tmp8q: Select.Operand = .{ .tag = .ref, .base = .tmp8q }; + const tmp8t: Select.Operand = .{ .tag = .ref, .base = .tmp8t }; + const tmp8x: Select.Operand = .{ .tag = .ref, .base = .tmp8x }; + const tmp8y: Select.Operand = .{ .tag = .ref, .base = .tmp8y }; + const dst0b: Select.Operand = .{ .tag = .ref, .base = .dst0b }; const dst0w: Select.Operand = .{ .tag = .ref, .base = .dst0w }; const dst0d: Select.Operand = .{ .tag = .ref, .base = .dst0d }; @@ -32204,14 +35480,20 @@ fn select( const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; caller_preserved: { - switch (switch (case.clobbers.caller_preserved) { + const cc = switch (case.clobbers.caller_preserved) { .none => break :caller_preserved, .ccc => cg.target.cCallingConvention().?, .zigcc => .auto, - }) { + }; + const err_ret_trace_reg = if (cc == .auto and cg.pt.zcu.comp.config.any_error_tracing) err_ret_trace_reg: { + const param_gpr = abi.getCAbiIntParamRegs(.auto); + break :err_ret_trace_reg param_gpr[param_gpr.len - 1]; + } else .none; + switch (cc) { else => unreachable, - inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| { - const tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse continue; + inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| skip: { + if (reg == err_ret_trace_reg) break :skip; + const tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse break :skip; try cg.register_manager.getRegIndex(tracked_index, null); assert(cg.register_manager.lockRegIndexAssumeUnused(tracked_index).tracked_index == tracked_index); }, @@ -32220,7 +35502,9 @@ fn select( @memcpy(src_slots[0..src_temps.len], src_temps); std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]); - for (tmp_slots, case.extra_temps) |*slot, spec| slot.* = try spec.create(&s) orelse continue; + for (1..3) |pass| for (tmp_slots, case.extra_temps) |*slot, spec| if (pass == spec.pass()) { + slot.* = try spec.create(&s); + }; while (true) for (pattern.src[0..src_temps.len], src_temps) |src_pattern, *src_temp| { if (try src_pattern.convert(src_temp, cg)) break; @@ -32231,7 +35515,7 @@ fn select( if (case.clobbers.eflags) try cg.spillEflagsIfOccupied(); for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, dst_kind| - dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)).?; + dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)); @memcpy(dst_slots[0..dst_temps.len], dst_temps); switch (case.each) { @@ -32243,16 +35527,20 @@ fn select( assert(s.top == 0); caller_preserved: { - switch (switch (case.clobbers.caller_preserved) { + const cc = switch (case.clobbers.caller_preserved) { .none => break :caller_preserved, .ccc => cg.target.cCallingConvention().?, .zigcc => .auto, - }) { + }; + const err_ret_trace_reg = if (cc == .auto and cg.pt.zcu.comp.config.any_error_tracing) err_ret_trace_reg: { + const param_gpr = abi.getCAbiIntParamRegs(.auto); + break :err_ret_trace_reg param_gpr[param_gpr.len - 1]; + } else .none; + switch (cc) { else => unreachable, - inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| { - cg.register_manager.unlockReg(.{ - .tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse continue, - }); + inline .x86_64_sysv, .x86_64_win, .auto => |_, tag| inline for (comptime abi.getCallerPreservedRegs(tag)) |reg| skip: { + if (reg == err_ret_trace_reg) break :skip; + cg.register_manager.unlockReg(.{ .tracked_index = RegisterManager.indexOfKnownRegIntoTracked(reg) orelse break :skip }); }, } } diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 9ab90e9881..8aae83d4c2 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -366,6 +366,7 @@ pub const Register = enum(u8) { @intFromEnum(Register.eax) ... @intFromEnum(Register.r15d) => @intFromEnum(Register.eax), @intFromEnum(Register.ax) ... @intFromEnum(Register.r15w) => @intFromEnum(Register.ax), @intFromEnum(Register.al) ... @intFromEnum(Register.r15b) => @intFromEnum(Register.al), + @intFromEnum(Register.ah) ... @intFromEnum(Register.bh) => @intFromEnum(Register.ah), else => unreachable, // zig fmt: on }; diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 5364b0432d..6244bd2324 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -706,7 +706,7 @@ pub const DeclGen = struct { const uav_ty = uav_val.typeOf(zcu); // Render an undefined pointer if we have a pointer to a zero-bit or comptime type. - const ptr_ty = Type.fromInterned(uav.orig_ty); + const ptr_ty: Type = .fromInterned(uav.orig_ty); if (ptr_ty.isPtrAtRuntime(zcu) and !uav_ty.isFnOrHasRuntimeBits(zcu)) { return dg.writeCValue(writer, .{ .undef = ptr_ty }); } @@ -782,7 +782,7 @@ pub const DeclGen = struct { }; // Render an undefined pointer if we have a pointer to a zero-bit or comptime type. - const nav_ty = Type.fromInterned(ip.getNav(owner_nav).typeOf(ip)); + const nav_ty: Type = .fromInterned(ip.getNav(owner_nav).typeOf(ip)); const ptr_ty = try pt.navPtrType(owner_nav); if (!nav_ty.isFnOrHasRuntimeBits(zcu)) { return dg.writeCValue(writer, .{ .undef = ptr_ty }); @@ -819,7 +819,7 @@ pub const DeclGen = struct { .comptime_alloc_ptr, .comptime_field_ptr => unreachable, .int => |int| { const ptr_ctype = try dg.ctypeFromType(int.ptr_ty, .complete); - const addr_val = try pt.intValue(Type.usize, int.addr); + const addr_val = try pt.intValue(.usize, int.addr); try writer.writeByte('('); try dg.renderCType(writer, ptr_ctype); try writer.print("){x}", .{try dg.fmtIntLiteral(addr_val, .Other)}); @@ -859,7 +859,7 @@ pub const DeclGen = struct { try writer.writeByte('('); try dg.renderCType(writer, ptr_ctype); try writer.writeByte(')'); - const offset_val = try pt.intValue(Type.usize, byte_offset); + const offset_val = try pt.intValue(.usize, byte_offset); try writer.writeAll("((char *)"); try dg.renderPointer(writer, field.parent.*, location); try writer.print(" + {})", .{try dg.fmtIntLiteral(offset_val, .Other)}); @@ -875,7 +875,7 @@ pub const DeclGen = struct { try writer.writeByte(')'); try dg.renderPointer(writer, elem.parent.*, location); } else { - const index_val = try pt.intValue(Type.usize, elem.elem_idx); + const index_val = try pt.intValue(.usize, elem.elem_idx); // We want to do pointer arithmetic on a pointer to the element type. // We might have a pointer-to-array. In this case, we must cast first. const result_ctype = try dg.ctypeFromType(elem.result_ptr_ty, .complete); @@ -904,7 +904,7 @@ pub const DeclGen = struct { if (oac.byte_offset == 0) { try dg.renderPointer(writer, oac.parent.*, location); } else { - const offset_val = try pt.intValue(Type.usize, oac.byte_offset); + const offset_val = try pt.intValue(.usize, oac.byte_offset); try writer.writeAll("((char *)"); try dg.renderPointer(writer, oac.parent.*, location); try writer.print(" + {})", .{try dg.fmtIntLiteral(offset_val, .Other)}); @@ -984,7 +984,7 @@ pub const DeclGen = struct { try writer.writeAll("(("); try dg.renderCType(writer, ctype); try writer.print("){x})", .{try dg.fmtIntLiteral( - try pt.intValue(Type.usize, val.toUnsignedInt(zcu)), + try pt.intValue(.usize, val.toUnsignedInt(zcu)), .Other, )}); }, @@ -1153,7 +1153,7 @@ pub const DeclGen = struct { else => |payload| switch (ip.indexToKey(payload)) { .undef => |err_ty| try dg.renderUndefValue( writer, - Type.fromInterned(err_ty), + .fromInterned(err_ty), location, ), .err => |err| try dg.renderErrorName(writer, err.name), @@ -1204,7 +1204,7 @@ pub const DeclGen = struct { ), }, .ptr => try writer.writeAll("NULL"), - .len => try dg.renderUndefValue(writer, Type.usize, initializer_type), + .len => try dg.renderUndefValue(writer, .usize, initializer_type), else => unreachable, } } @@ -1219,7 +1219,7 @@ pub const DeclGen = struct { try writer.writeByte(')'); } const ai = ty.arrayInfo(zcu); - if (ai.elem_type.eql(Type.u8, zcu)) { + if (ai.elem_type.eql(.u8, zcu)) { var literal = stringLiteral(writer, ty.arrayLenIncludingSentinel(zcu)); try literal.start(); var index: usize = 0; @@ -1263,7 +1263,7 @@ pub const DeclGen = struct { for (0..tuple.types.len) |field_index| { const comptime_val = tuple.values.get(ip)[field_index]; if (comptime_val != .none) continue; - const field_ty = Type.fromInterned(tuple.types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(tuple.types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; if (!empty) try writer.writeByte(','); @@ -1298,7 +1298,7 @@ pub const DeclGen = struct { var field_it = loaded_struct.iterateRuntimeOrder(ip); var need_comma = false; while (field_it.next()) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; if (need_comma) try writer.writeByte(','); @@ -1325,7 +1325,7 @@ pub const DeclGen = struct { var eff_num_fields: usize = 0; for (0..loaded_struct.field_types.len) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; eff_num_fields += 1; } @@ -1346,7 +1346,7 @@ pub const DeclGen = struct { var eff_index: usize = 0; var needs_closing_paren = false; for (0..loaded_struct.field_types.len) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) { @@ -1382,7 +1382,7 @@ pub const DeclGen = struct { // a << a_off | b << b_off | c << c_off var empty = true; for (0..loaded_struct.field_types.len) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; if (!empty) try writer.writeAll(" | "); @@ -1466,7 +1466,7 @@ pub const DeclGen = struct { } const field_index = zcu.unionTagFieldIndex(loaded_union, Value.fromInterned(un.tag)).?; - const field_ty = Type.fromInterned(loaded_union.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]); const field_name = loaded_union.loadTagType(ip).names.get(ip)[field_index]; if (loaded_union.flagsUnordered(ip).layout == .@"packed") { if (field_ty.hasRuntimeBits(zcu)) { @@ -1509,7 +1509,7 @@ pub const DeclGen = struct { ); try writer.writeByte(' '); } else for (0..loaded_union.field_types.len) |inner_field_index| { - const inner_field_ty = Type.fromInterned( + const inner_field_ty: Type = .fromInterned( loaded_union.field_types.get(ip)[inner_field_index], ); if (!inner_field_ty.hasRuntimeBits(zcu)) continue; @@ -1592,7 +1592,7 @@ pub const DeclGen = struct { try writer.writeAll("(("); try dg.renderCType(writer, ctype); return writer.print("){x})", .{ - try dg.fmtIntLiteral(try pt.undefValue(Type.usize), .Other), + try dg.fmtIntLiteral(try pt.undefValue(.usize), .Other), }); }, .slice => { @@ -1606,14 +1606,14 @@ pub const DeclGen = struct { const ptr_ty = ty.slicePtrFieldType(zcu); try dg.renderType(writer, ptr_ty); return writer.print("){x}, {0x}}}", .{ - try dg.fmtIntLiteral(try dg.pt.undefValue(Type.usize), .Other), + try dg.fmtIntLiteral(try dg.pt.undefValue(.usize), .Other), }); }, }, .opt_type => |child_type| switch (ctype.info(ctype_pool)) { .basic, .pointer => try dg.renderUndefValue( writer, - Type.fromInterned(if (ctype.isBool()) .bool_type else child_type), + .fromInterned(if (ctype.isBool()) .bool_type else child_type), location, ), .aligned, .array, .vector, .fwd_decl, .function => unreachable, @@ -1622,7 +1622,7 @@ pub const DeclGen = struct { .is_null, .payload => {}, .ptr, .len => return dg.renderUndefValue( writer, - Type.fromInterned(child_type), + .fromInterned(child_type), location, ), else => unreachable, @@ -1635,7 +1635,7 @@ pub const DeclGen = struct { try writer.writeByte('{'); for (0..aggregate.fields.len) |field_index| { if (field_index > 0) try writer.writeByte(','); - try dg.renderUndefValue(writer, Type.fromInterned( + try dg.renderUndefValue(writer, .fromInterned( switch (aggregate.fields.at(field_index, ctype_pool).name.index) { .is_null => .bool_type, .payload => child_type, @@ -1660,7 +1660,7 @@ pub const DeclGen = struct { var field_it = loaded_struct.iterateRuntimeOrder(ip); var need_comma = false; while (field_it.next()) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; if (need_comma) try writer.writeByte(','); @@ -1685,7 +1685,7 @@ pub const DeclGen = struct { var need_comma = false; for (0..tuple_info.types.len) |field_index| { if (tuple_info.values.get(ip)[field_index] != .none) continue; - const field_ty = Type.fromInterned(tuple_info.types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(tuple_info.types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; if (need_comma) try writer.writeByte(','); @@ -1715,13 +1715,13 @@ pub const DeclGen = struct { .payload) { .tag => try dg.renderUndefValue( writer, - Type.fromInterned(loaded_union.enum_tag_ty), + .fromInterned(loaded_union.enum_tag_ty), initializer_type, ), .payload => { try writer.writeByte('{'); for (0..loaded_union.field_types.len) |inner_field_index| { - const inner_field_ty = Type.fromInterned( + const inner_field_ty: Type = .fromInterned( loaded_union.field_types.get(ip)[inner_field_index], ); if (!inner_field_ty.hasRuntimeBits(pt.zcu)) continue; @@ -1747,7 +1747,7 @@ pub const DeclGen = struct { .error_union_type => |error_union_type| switch (ctype.info(ctype_pool)) { .basic => try dg.renderUndefValue( writer, - Type.fromInterned(error_union_type.error_set_type), + .fromInterned(error_union_type.error_set_type), location, ), .pointer, .aligned, .array, .vector, .fwd_decl, .function => unreachable, @@ -1762,7 +1762,7 @@ pub const DeclGen = struct { if (field_index > 0) try writer.writeByte(','); try dg.renderUndefValue( writer, - Type.fromInterned( + .fromInterned( switch (aggregate.fields.at(field_index, ctype_pool).name.index) { .@"error" => error_union_type.error_set_type, .payload => error_union_type.payload_type, @@ -1777,7 +1777,7 @@ pub const DeclGen = struct { }, .array_type, .vector_type => { const ai = ty.arrayInfo(zcu); - if (ai.elem_type.eql(Type.u8, zcu)) { + if (ai.elem_type.eql(.u8, zcu)) { const c_len = ty.arrayLenIncludingSentinel(zcu); var literal = stringLiteral(writer, c_len); try literal.start(); @@ -1981,8 +1981,8 @@ pub const DeclGen = struct { const src_is_ptr = src_ty.isPtrAtRuntime(pt.zcu); const src_eff_ty: Type = if (src_is_ptr) switch (dest_int_info.signedness) { - .unsigned => Type.usize, - .signed => Type.isize, + .unsigned => .usize, + .signed => .isize, } else src_ty; const src_bits = src_eff_ty.bitSize(zcu); @@ -2022,8 +2022,8 @@ pub const DeclGen = struct { const src_is_ptr = src_ty.isPtrAtRuntime(zcu); const src_eff_ty: Type = if (src_is_ptr) switch (dest_int_info.signedness) { - .unsigned => Type.usize, - .signed => Type.isize, + .unsigned => .usize, + .signed => .isize, } else src_ty; const src_bits = src_eff_ty.bitSize(zcu); @@ -2249,7 +2249,7 @@ pub const DeclGen = struct { if (flags.is_threadlocal and !dg.mod.single_threaded) try fwd.writeAll("zig_threadlocal "); try dg.renderTypeAndName( fwd, - Type.fromInterned(nav.typeOf(ip)), + .fromInterned(nav.typeOf(ip)), .{ .nav = nav_index }, CQualifiers.init(.{ .@"const" = flags.is_const }), nav.getAlignment(), @@ -2321,7 +2321,7 @@ pub const DeclGen = struct { if (is_big) try writer.print(", {}", .{int_info.signedness == .signed}); try writer.print(", {}", .{try dg.fmtIntLiteral( - try pt.intValue(if (is_big) Type.u16 else Type.u8, int_info.bits), + try pt.intValue(if (is_big) .u16 else .u8, int_info.bits), .FunctionArgument, )}); } @@ -2657,7 +2657,7 @@ pub fn genTypeDecl( }, .index => |index| if (!found_existing) { const ip = &zcu.intern_pool; - const ty = Type.fromInterned(index); + const ty: Type = .fromInterned(index); _ = try renderTypePrefix(.flush, global_ctype_pool, zcu, writer, global_ctype, .suffix, .{}); try writer.writeByte(';'); const file_scope = ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip); @@ -2772,7 +2772,7 @@ pub fn genErrDecls(o: *Object) !void { if (val > 1) try writer.writeAll(", "); try writer.print("{{" ++ name_prefix ++ "{}, {}}}", .{ fmtIdent(name), - try o.dg.fmtIntLiteral(try pt.intValue(Type.usize, name.len), .StaticInitializer), + try o.dg.fmtIntLiteral(try pt.intValue(.usize, name.len), .StaticInitializer), }); } try writer.writeAll("};\n"); @@ -2788,8 +2788,8 @@ pub fn genLazyFn(o: *Object, lazy_ctype_pool: *const CType.Pool, lazy_fn: LazyFn const val = lazy_fn.value_ptr; switch (key) { .tag_name => |enum_ty_ip| { - const enum_ty = Type.fromInterned(enum_ty_ip); - const name_slice_ty = Type.slice_const_u8_sentinel_0; + const enum_ty: Type = .fromInterned(enum_ty_ip); + const name_slice_ty: Type = .slice_const_u8_sentinel_0; try w.writeAll("static "); try o.dg.renderType(w, name_slice_ty); @@ -2822,7 +2822,7 @@ pub fn genLazyFn(o: *Object, lazy_ctype_pool: *const CType.Pool, lazy_fn: LazyFn try o.dg.renderType(w, name_slice_ty); try w.print("){{{}, {}}};\n", .{ fmtIdent("name"), - try o.dg.fmtIntLiteral(try pt.intValue(Type.usize, tag_name_len), .Other), + try o.dg.fmtIntLiteral(try pt.intValue(.usize, tag_name_len), .Other), }); try w.writeAll(" }\n"); @@ -2966,7 +2966,7 @@ pub fn genDecl(o: *Object) !void { const zcu = pt.zcu; const ip = &zcu.intern_pool; const nav = ip.getNav(o.dg.pass.nav); - const nav_ty = Type.fromInterned(nav.typeOf(ip)); + const nav_ty: Type = .fromInterned(nav.typeOf(ip)); if (!nav_ty.isFnOrHasRuntimeBitsIgnoreComptime(zcu)) return; switch (ip.indexToKey(nav.status.fully_resolved.val)) { @@ -3717,7 +3717,7 @@ fn airArg(f: *Function, inst: Air.Inst.Index) !CValue { if (f.liveness.isUnused(inst)) { const writer = f.object.writer(); try writer.writeByte('('); - try f.renderType(writer, Type.void); + try f.renderType(writer, .void); try writer.writeByte(')'); try f.writeCValue(writer, result, .Other); try writer.writeAll(";\n"); @@ -3735,7 +3735,7 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { const ptr_ty = f.typeOf(ty_op.operand); const ptr_scalar_ty = ptr_ty.scalarType(zcu); const ptr_info = ptr_scalar_ty.ptrInfo(zcu); - const src_ty = Type.fromInterned(ptr_info.child); + const src_ty: Type = .fromInterned(ptr_info.child); if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) { try reap(f, inst, &.{ty_op.operand}); @@ -3953,7 +3953,7 @@ fn airTrunc(f: *Function, inst: Air.Inst.Index) !CValue { .signed => { const c_bits = toCIntBits(scalar_int_info.bits) orelse return f.fail("TODO: C backend: implement integer types larger than 128 bits", .{}); - const shift_val = try pt.intValue(Type.u8, c_bits - dest_bits); + const shift_val = try pt.intValue(.u8, c_bits - dest_bits); try writer.writeAll("zig_shr_"); try f.object.dg.renderTypeForBuiltinFnName(writer, scalar_ty); @@ -4019,7 +4019,7 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { try writer.writeAll("memset("); try f.writeCValue(writer, ptr_val, .FunctionArgument); try writer.writeAll(", 0xaa, sizeof("); - try f.renderType(writer, Type.fromInterned(ptr_info.child)); + try f.renderType(writer, .fromInterned(ptr_info.child)); try writer.writeAll("));\n"); } return .none; @@ -4029,7 +4029,7 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { ptr_info.flags.alignment.order(src_ty.abiAlignment(zcu)).compare(.gte) else true; - const is_array = lowersToArray(Type.fromInterned(ptr_info.child), pt); + const is_array = lowersToArray(.fromInterned(ptr_info.child), pt); const need_memcpy = !is_aligned or is_array; const src_val = try f.resolveInst(bin_op.rhs); @@ -4040,7 +4040,7 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { if (need_memcpy) { // For this memcpy to safely work we need the rhs to have the same // underlying type as the lhs (i.e. they must both be arrays of the same underlying type). - assert(src_ty.eql(Type.fromInterned(ptr_info.child), zcu)); + assert(src_ty.eql(.fromInterned(ptr_info.child), zcu)); // If the source is a constant, writeCValue will emit a brace initialization // so work around this by initializing into new local. @@ -4120,7 +4120,7 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { if (src_ty.isPtrAtRuntime(zcu)) { try writer.writeByte('('); - try f.renderType(writer, Type.usize); + try f.renderType(writer, .usize); try writer.writeByte(')'); } try f.writeCValue(writer, src_val, .Other); @@ -4343,8 +4343,8 @@ fn airEquality( try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); - const local = try f.allocLocal(inst, Type.bool); - const a = try Assignment.start(f, writer, CType.bool); + const local = try f.allocLocal(inst, .bool); + const a = try Assignment.start(f, writer, .bool); try f.writeCValue(writer, local, .Other); try a.assign(f, writer); @@ -4401,7 +4401,7 @@ fn airCmpLtErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue { try reap(f, inst, &.{un_op}); const writer = f.object.writer(); - const local = try f.allocLocal(inst, Type.bool); + const local = try f.allocLocal(inst, .bool); try f.writeCValue(writer, local, .Other); try writer.writeAll(" = "); try f.writeCValue(writer, operand, .Other); @@ -4517,7 +4517,7 @@ fn airSlice(f: *Function, inst: Air.Inst.Index) !CValue { try a.end(f, writer); } { - const a = try Assignment.start(f, writer, CType.usize); + const a = try Assignment.start(f, writer, .usize); try f.writeCValueMember(writer, local, .{ .identifier = "len" }); try a.assign(f, writer); try f.writeCValue(writer, len, .Initializer); @@ -4585,9 +4585,9 @@ fn airCall( else => unreachable, }; const fn_info = zcu.typeToFunc(if (callee_is_ptr) callee_ty.childType(zcu) else callee_ty).?; - const ret_ty = Type.fromInterned(fn_info.return_type); + const ret_ty: Type = .fromInterned(fn_info.return_type); const ret_ctype: CType = if (ret_ty.isNoReturn(zcu)) - CType.void + .void else try f.ctypeFromType(ret_ty, .parameter); @@ -4599,7 +4599,7 @@ fn airCall( break :result .none; } else if (f.liveness.isUnused(inst)) { try writer.writeByte('('); - try f.renderCType(writer, CType.void); + try f.renderCType(writer, .void); try writer.writeByte(')'); break :result .none; } else { @@ -5081,20 +5081,20 @@ fn airBreakpoint(writer: anytype) !CValue { fn airRetAddr(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); - const local = try f.allocLocal(inst, Type.usize); + const local = try f.allocLocal(inst, .usize); try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); - try f.renderType(writer, Type.usize); + try f.renderType(writer, .usize); try writer.writeAll(")zig_return_address();\n"); return local; } fn airFrameAddress(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); - const local = try f.allocLocal(inst, Type.usize); + const local = try f.allocLocal(inst, .usize); try f.writeCValue(writer, local, .Other); try writer.writeAll(" = ("); - try f.renderType(writer, Type.usize); + try f.renderType(writer, .usize); try writer.writeAll(")zig_frame_address();\n"); return local; } @@ -5179,10 +5179,10 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index, is_dispatch_loop: bool) !void try writer.writeAll("switch ("); - const lowered_condition_ty = if (condition_ty.toIntern() == .bool_type) - Type.u1 + const lowered_condition_ty: Type = if (condition_ty.toIntern() == .bool_type) + .u1 else if (condition_ty.isPtrAtRuntime(zcu)) - Type.usize + .usize else condition_ty; if (condition_ty.toIntern() != lowered_condition_ty.toIntern()) { @@ -5219,7 +5219,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index, is_dispatch_loop: bool) !void } if (condition_ty.isPtrAtRuntime(zcu)) { try writer.writeByte('('); - try f.renderType(writer, Type.usize); + try f.renderType(writer, .usize); try writer.writeByte(')'); } try f.object.dg.renderValue(writer, (try f.air.value(item, pt)).?, .Other); @@ -5604,8 +5604,8 @@ fn airIsNull( const operand = try f.resolveInst(un_op); try reap(f, inst, &.{un_op}); - const local = try f.allocLocal(inst, Type.bool); - const a = try Assignment.start(f, writer, CType.bool); + const local = try f.allocLocal(inst, .bool); + const a = try Assignment.start(f, writer, .bool); try f.writeCValue(writer, local, .Other); try a.assign(f, writer); @@ -5750,7 +5750,7 @@ fn fieldLocation( } { const zcu = pt.zcu; const ip = &zcu.intern_pool; - const container_ty = Type.fromInterned(ip.indexToKey(container_ptr_ty.toIntern()).ptr_type.child); + const container_ty: Type = .fromInterned(ip.indexToKey(container_ptr_ty.toIntern()).ptr_type.child); switch (ip.indexToKey(container_ty.toIntern())) { .struct_type => { const loaded_struct = ip.loadStructType(container_ty.toIntern()); @@ -5781,7 +5781,7 @@ fn fieldLocation( const loaded_union = ip.loadUnionType(container_ty.toIntern()); switch (loaded_union.flagsUnordered(ip).layout) { .auto, .@"extern" => { - const field_ty = Type.fromInterned(loaded_union.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) return if (loaded_union.hasTag(ip) and !container_ty.unionHasAllZeroBitFieldTypes(zcu)) .{ .field = .{ .identifier = "payload" } } @@ -5850,7 +5850,7 @@ fn airFieldParentPtr(f: *Function, inst: Air.Inst.Index) !CValue { switch (fieldLocation(container_ptr_ty, field_ptr_ty, extra.field_index, pt)) { .begin => try f.writeCValue(writer, field_ptr_val, .Initializer), .field => |field| { - const u8_ptr_ty = try pt.adjustPtrTypeChild(field_ptr_ty, Type.u8); + const u8_ptr_ty = try pt.adjustPtrTypeChild(field_ptr_ty, .u8); try writer.writeAll("(("); try f.renderType(writer, u8_ptr_ty); @@ -5863,14 +5863,14 @@ fn airFieldParentPtr(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll("))"); }, .byte_offset => |byte_offset| { - const u8_ptr_ty = try pt.adjustPtrTypeChild(field_ptr_ty, Type.u8); + const u8_ptr_ty = try pt.adjustPtrTypeChild(field_ptr_ty, .u8); try writer.writeAll("(("); try f.renderType(writer, u8_ptr_ty); try writer.writeByte(')'); try f.writeCValue(writer, field_ptr_val, .Other); try writer.print(" - {})", .{ - try f.fmtIntLiteral(try pt.intValue(Type.usize, byte_offset)), + try f.fmtIntLiteral(try pt.intValue(.usize, byte_offset)), }); }, } @@ -5908,14 +5908,14 @@ fn fieldPtr( try f.writeCValueDerefMember(writer, container_ptr_val, field); }, .byte_offset => |byte_offset| { - const u8_ptr_ty = try pt.adjustPtrTypeChild(field_ptr_ty, Type.u8); + const u8_ptr_ty = try pt.adjustPtrTypeChild(field_ptr_ty, .u8); try writer.writeAll("(("); try f.renderType(writer, u8_ptr_ty); try writer.writeByte(')'); try f.writeCValue(writer, container_ptr_val, .Other); try writer.print(" + {})", .{ - try f.fmtIntLiteral(try pt.intValue(Type.usize, byte_offset)), + try f.fmtIntLiteral(try pt.intValue(.usize, byte_offset)), }); }, } @@ -6158,7 +6158,7 @@ fn airWrapOptional(f: *Function, inst: Air.Inst.Index) !CValue { const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); { - const a = try Assignment.start(f, writer, CType.bool); + const a = try Assignment.start(f, writer, .bool); try f.writeCValueMember(writer, local, .{ .identifier = "is_null" }); try a.assign(f, writer); try writer.writeAll("false"); @@ -6322,12 +6322,12 @@ fn airIsErr(f: *Function, inst: Air.Inst.Index, is_ptr: bool, operator: []const const operand = try f.resolveInst(un_op); try reap(f, inst, &.{un_op}); const operand_ty = f.typeOf(un_op); - const local = try f.allocLocal(inst, Type.bool); + const local = try f.allocLocal(inst, .bool); const err_union_ty = if (is_ptr) operand_ty.childType(zcu) else operand_ty; const payload_ty = err_union_ty.errorUnionPayload(zcu); const error_ty = err_union_ty.errorUnionSet(zcu); - const a = try Assignment.start(f, writer, CType.bool); + const a = try Assignment.start(f, writer, .bool); try f.writeCValue(writer, local, .Other); try a.assign(f, writer); const err_int_ty = try pt.errorIntType(); @@ -6385,17 +6385,17 @@ fn airArrayToSlice(f: *Function, inst: Air.Inst.Index) !CValue { if (operand_child_ctype.info(ctype_pool) == .array) { try writer.writeByte('&'); try f.writeCValueDeref(writer, operand); - try writer.print("[{}]", .{try f.fmtIntLiteral(try pt.intValue(Type.usize, 0))}); + try writer.print("[{}]", .{try f.fmtIntLiteral(try pt.intValue(.usize, 0))}); } else try f.writeCValue(writer, operand, .Initializer); } try a.end(f, writer); } { - const a = try Assignment.start(f, writer, CType.usize); + const a = try Assignment.start(f, writer, .usize); try f.writeCValueMember(writer, local, .{ .identifier = "len" }); try a.assign(f, writer); try writer.print("{}", .{ - try f.fmtIntLiteral(try pt.intValue(Type.usize, array_ty.arrayLen(zcu))), + try f.fmtIntLiteral(try pt.intValue(.usize, array_ty.arrayLen(zcu))), }); try a.end(f, writer); } @@ -6627,7 +6627,7 @@ fn airCmpBuiltinCall( try writer.writeByte(')'); if (!ref_ret) try writer.print("{s}{}", .{ compareOperatorC(operator), - try f.fmtIntLiteral(try pt.intValue(Type.i32, 0)), + try f.fmtIntLiteral(try pt.intValue(.i32, 0)), }); try writer.writeAll(";\n"); try v.end(f, inst, writer); @@ -6707,7 +6707,7 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue try a.end(f, writer); } { - const a = try Assignment.start(f, writer, CType.bool); + const a = try Assignment.start(f, writer, .bool); try f.writeCValueMember(writer, local, .{ .identifier = "is_null" }); try a.assign(f, writer); try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor}); @@ -6935,12 +6935,12 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { }, }); - const index = try f.allocLocal(inst, Type.usize); + const index = try f.allocLocal(inst, .usize); try writer.writeAll("for ("); try f.writeCValue(writer, index, .Other); try writer.writeAll(" = "); - try f.object.dg.renderValue(writer, try pt.intValue(Type.usize, 0), .Initializer); + try f.object.dg.renderValue(writer, try pt.intValue(.usize, 0), .Initializer); try writer.writeAll("; "); try f.writeCValue(writer, index, .Other); try writer.writeAll(" != "); @@ -6976,7 +6976,7 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { return .none; } - const bitcasted = try bitcast(f, Type.u8, value, elem_ty); + const bitcasted = try bitcast(f, .u8, value, elem_ty); try writer.writeAll("memset("); switch (dest_ty.ptrSize(zcu)) { @@ -7038,7 +7038,7 @@ fn writeArrayLen(f: *Function, writer: ArrayListWriter, dest_ptr: CValue, dest_t const zcu = pt.zcu; switch (dest_ty.ptrSize(zcu)) { .one => try writer.print("{}", .{ - try f.fmtIntLiteral(try pt.intValue(Type.usize, dest_ty.childType(zcu).arrayLen(zcu))), + try f.fmtIntLiteral(try pt.intValue(.usize, dest_ty.childType(zcu).arrayLen(zcu))), }), .many, .c => unreachable, .slice => try f.writeCValueMember(writer, dest_ptr, .{ .identifier = "len" }), @@ -7200,11 +7200,11 @@ fn airShuffle(f: *Function, inst: Air.Inst.Index) !CValue { for (0..extra.mask_len) |index| { try f.writeCValue(writer, local, .Other); try writer.writeByte('['); - try f.object.dg.renderValue(writer, try pt.intValue(Type.usize, index), .Other); + try f.object.dg.renderValue(writer, try pt.intValue(.usize, index), .Other); try writer.writeAll("] = "); const mask_elem = (try mask.elemValue(pt, index)).toSignedInt(zcu); - const src_val = try pt.intValue(Type.usize, @as(u64, @intCast(mask_elem ^ mask_elem >> 63))); + const src_val = try pt.intValue(.usize, @as(u64, @intCast(mask_elem ^ mask_elem >> 63))); try f.writeCValue(writer, if (mask_elem >= 0) lhs else rhs, .Other); try writer.writeByte('['); @@ -7377,7 +7377,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { switch (ip.indexToKey(inst_ty.toIntern())) { inline .array_type, .vector_type => |info, tag| { const a: Assignment = .{ - .ctype = try f.ctypeFromType(Type.fromInterned(info.child), .complete), + .ctype = try f.ctypeFromType(.fromInterned(info.child), .complete), }; for (resolved_elements, 0..) |element, i| { try a.restart(f, writer); @@ -7402,7 +7402,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { .auto, .@"extern" => { var field_it = loaded_struct.iterateRuntimeOrder(ip); while (field_it.next()) |field_index| { - const field_ty = Type.fromInterned(loaded_struct.field_types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; const a = try Assignment.start(f, writer, try f.ctypeFromType(field_ty, .complete)); @@ -7466,8 +7466,8 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { if (field_ty.isPtrAtRuntime(zcu)) { try writer.writeByte('('); try f.renderType(writer, switch (int_info.signedness) { - .unsigned => Type.usize, - .signed => Type.isize, + .unsigned => .usize, + .signed => .isize, }); try writer.writeByte(')'); } @@ -7501,7 +7501,7 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { }, .tuple_type => |tuple_info| for (0..tuple_info.types.len) |field_index| { if (tuple_info.values.get(ip)[field_index] != .none) continue; - const field_ty = Type.fromInterned(tuple_info.types.get(ip)[field_index]); + const field_ty: Type = .fromInterned(tuple_info.types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; const a = try Assignment.start(f, writer, try f.ctypeFromType(field_ty, .complete)); @@ -8141,13 +8141,13 @@ fn formatIntLiteral( } = switch (data.ctype.info(ctype_pool)) { .basic => |basic_info| switch (basic_info) { else => .{ - .ctype = CType.void, + .ctype = .void, .count = 1, .endian = .little, .homogeneous = true, }, .zig_u128, .zig_i128 => .{ - .ctype = CType.u64, + .ctype = .u64, .count = 2, .endian = .big, .homogeneous = false, @@ -8253,7 +8253,7 @@ fn formatIntLiteral( .int_info = c_limb_int_info, .kind = data.kind, .ctype = c_limb_ctype, - .val = try pt.intValue_big(Type.comptime_int, c_limb_mut.toConst()), + .val = try pt.intValue_big(.comptime_int, c_limb_mut.toConst()), }, fmt, options, writer); } } @@ -8330,15 +8330,15 @@ const Vectorize = struct { const pt = f.object.dg.pt; const zcu = pt.zcu; return if (ty.zigTypeTag(zcu) == .vector) index: { - const local = try f.allocLocal(inst, Type.usize); + const local = try f.allocLocal(inst, .usize); try writer.writeAll("for ("); try f.writeCValue(writer, local, .Other); - try writer.print(" = {d}; ", .{try f.fmtIntLiteral(try pt.intValue(Type.usize, 0))}); + try writer.print(" = {d}; ", .{try f.fmtIntLiteral(try pt.intValue(.usize, 0))}); try f.writeCValue(writer, local, .Other); - try writer.print(" < {d}; ", .{try f.fmtIntLiteral(try pt.intValue(Type.usize, ty.vectorLen(zcu)))}); + try writer.print(" < {d}; ", .{try f.fmtIntLiteral(try pt.intValue(.usize, ty.vectorLen(zcu)))}); try f.writeCValue(writer, local, .Other); - try writer.print(" += {d}) {{\n", .{try f.fmtIntLiteral(try pt.intValue(Type.usize, 1))}); + try writer.print(" += {d}) {{\n", .{try f.fmtIntLiteral(try pt.intValue(.usize, 1))}); f.object.indent_writer.pushIndent(); break :index .{ .index = local }; diff --git a/src/codegen/c/Type.zig b/src/codegen/c/Type.zig index 2673576b90..4aacc7fa5d 100644 --- a/src/codegen/c/Type.zig +++ b/src/codegen/c/Type.zig @@ -1289,26 +1289,26 @@ pub const Pool = struct { kind: Kind, ) !CType { switch (int_info.bits) { - 0 => return CType.void, + 0 => return .void, 1...8 => switch (int_info.signedness) { - .signed => return CType.i8, - .unsigned => return CType.u8, + .signed => return .i8, + .unsigned => return .u8, }, 9...16 => switch (int_info.signedness) { - .signed => return CType.i16, - .unsigned => return CType.u16, + .signed => return .i16, + .unsigned => return .u16, }, 17...32 => switch (int_info.signedness) { - .signed => return CType.i32, - .unsigned => return CType.u32, + .signed => return .i32, + .unsigned => return .u32, }, 33...64 => switch (int_info.signedness) { - .signed => return CType.i64, - .unsigned => return CType.u64, + .signed => return .i64, + .unsigned => return .u64, }, 65...128 => switch (int_info.signedness) { - .signed => return CType.i128, - .unsigned => return CType.u128, + .signed => return .i128, + .unsigned => return .u128, }, else => { const target = &mod.resolved_target.result; @@ -1357,19 +1357,19 @@ pub const Pool = struct { .null_type, .undefined_type, .enum_literal_type, - => return CType.void, - .u1_type, .u8_type => return CType.u8, - .i8_type => return CType.i8, - .u16_type => return CType.u16, - .i16_type => return CType.i16, - .u29_type, .u32_type => return CType.u32, - .i32_type => return CType.i32, - .u64_type => return CType.u64, - .i64_type => return CType.i64, - .u80_type, .u128_type => return CType.u128, - .i128_type => return CType.i128, - .usize_type => return CType.usize, - .isize_type => return CType.isize, + => return .void, + .u1_type, .u8_type => return .u8, + .i8_type => return .i8, + .u16_type => return .u16, + .i16_type => return .i16, + .u29_type, .u32_type => return .u32, + .i32_type => return .i32, + .u64_type => return .u64, + .i64_type => return .i64, + .u80_type, .u128_type => return .u128, + .i128_type => return .i128, + .usize_type => return .usize, + .isize_type => return .isize, .c_char_type => return .{ .index = .char }, .c_short_type => return .{ .index = .short }, .c_ushort_type => return .{ .index = .@"unsigned short" }, @@ -1380,12 +1380,12 @@ pub const Pool = struct { .c_longlong_type => return .{ .index = .@"long long" }, .c_ulonglong_type => return .{ .index = .@"unsigned long long" }, .c_longdouble_type => return .{ .index = .@"long double" }, - .f16_type => return CType.f16, - .f32_type => return CType.f32, - .f64_type => return CType.f64, - .f80_type => return CType.f80, - .f128_type => return CType.f128, - .bool_type, .optional_noreturn_type => return CType.bool, + .f16_type => return .f16, + .f32_type => return .f32, + .f64_type => return .f64, + .f80_type => return .f80, + .f128_type => return .f128, + .bool_type, .optional_noreturn_type => return .bool, .noreturn_type, .anyframe_type, .generic_poison_type, @@ -1397,19 +1397,20 @@ pub const Pool = struct { .signedness = .unsigned, .bits = pt.zcu.errorSetBits(), }, mod, kind), + .manyptr_u8_type, => return pool.getPointer(allocator, .{ - .elem_ctype = CType.u8, + .elem_ctype = .u8, }), .manyptr_const_u8_type, .manyptr_const_u8_sentinel_0_type, => return pool.getPointer(allocator, .{ - .elem_ctype = CType.u8, + .elem_ctype = .u8, .@"const" = true, }), .single_const_pointer_to_comptime_int_type, => return pool.getPointer(allocator, .{ - .elem_ctype = CType.void, + .elem_ctype = .void, .@"const" = true, }), .slice_const_u8_type, @@ -1420,14 +1421,14 @@ pub const Pool = struct { .{ .name = .{ .index = .ptr }, .ctype = try pool.getPointer(allocator, .{ - .elem_ctype = CType.u8, + .elem_ctype = .u8, .@"const" = true, }), .alignas = AlignAs.fromAbiAlignment(Type.ptrAbiAlignment(target.*)), }, .{ .name = .{ .index = .len }, - .ctype = CType.usize, + .ctype = .usize, .alignas = AlignAs.fromAbiAlignment( Type.intAbiAlignment(target.ptrBitWidth(), target.*), ), @@ -1436,6 +1437,97 @@ pub const Pool = struct { return pool.fromFields(allocator, .@"struct", &fields, kind); }, + .vector_4_f16_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .f16, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.f16.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_8_f16_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .f16, + .len = 8, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.f16.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_4_f32_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .f32, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.f32.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_8_f32_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .f32, + .len = 8, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.f32.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_2_f64_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .f64, + .len = 2, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.f64.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_4_f64_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .f64, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.f64.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .undef, .zero, .zero_usize, @@ -1521,7 +1613,7 @@ pub const Pool = struct { }, .{ .name = .{ .index = .len }, - .ctype = CType.usize, + .ctype = .usize, .alignas = AlignAs.fromAbiAlignment( Type.intAbiAlignment(target.ptrBitWidth(), target.*), ), @@ -1532,7 +1624,7 @@ pub const Pool = struct { }, .array_type => |array_info| { const len = array_info.lenIncludingSentinel(); - if (len == 0) return CType.void; + if (len == 0) return .void; const elem_type = Type.fromInterned(array_info.child); const elem_ctype = try pool.fromType( allocator, @@ -1542,7 +1634,7 @@ pub const Pool = struct { mod, kind.noParameter(), ); - if (elem_ctype.index == .void) return CType.void; + if (elem_ctype.index == .void) return .void; const array_ctype = try pool.getArray(allocator, .{ .elem_ctype = elem_ctype, .len = len, @@ -1558,7 +1650,7 @@ pub const Pool = struct { return pool.fromFields(allocator, .@"struct", &fields, kind); }, .vector_type => |vector_info| { - if (vector_info.len == 0) return CType.void; + if (vector_info.len == 0) return .void; const elem_type = Type.fromInterned(vector_info.child); const elem_ctype = try pool.fromType( allocator, @@ -1568,7 +1660,7 @@ pub const Pool = struct { mod, kind.noParameter(), ); - if (elem_ctype.index == .void) return CType.void; + if (elem_ctype.index == .void) return .void; const vector_ctype = try pool.getVector(allocator, .{ .elem_ctype = elem_ctype, .len = vector_info.len, @@ -1584,7 +1676,7 @@ pub const Pool = struct { return pool.fromFields(allocator, .@"struct", &fields, kind); }, .opt_type => |payload_type| { - if (ip.isNoReturn(payload_type)) return CType.void; + if (ip.isNoReturn(payload_type)) return .void; const payload_ctype = try pool.fromType( allocator, scratch, @@ -1593,7 +1685,7 @@ pub const Pool = struct { mod, kind.noParameter(), ); - if (payload_ctype.index == .void) return CType.bool; + if (payload_ctype.index == .void) return .bool; switch (payload_type) { .anyerror_type => return payload_ctype, else => switch (ip.indexToKey(payload_type)) { @@ -1606,7 +1698,7 @@ pub const Pool = struct { var fields = [_]Info.Field{ .{ .name = .{ .index = .is_null }, - .ctype = CType.bool, + .ctype = .bool, .alignas = AlignAs.fromAbiAlignment(.@"1"), }, .{ @@ -1666,7 +1758,7 @@ pub const Pool = struct { if (kind.isForward()) return if (ty.hasRuntimeBitsIgnoreComptime(zcu)) fwd_decl else - CType.void; + .void; const scratch_top = scratch.items.len; defer scratch.shrinkRetainingCapacity(scratch_top); try scratch.ensureUnusedCapacity( @@ -1710,7 +1802,7 @@ pub const Pool = struct { scratch.items.len - scratch_top, @typeInfo(Field).@"struct".fields.len, )); - if (fields_len == 0) return CType.void; + if (fields_len == 0) return .void; try pool.ensureUnusedCapacity(allocator, 1); const extra_index = try pool.addHashedExtra(allocator, &hasher, Aggregate, .{ .fwd_decl = fwd_decl.index, @@ -1762,7 +1854,7 @@ pub const Pool = struct { scratch.items.len - scratch_top, @typeInfo(Field).@"struct".fields.len, )); - if (fields_len == 0) return CType.void; + if (fields_len == 0) return .void; if (kind.isForward()) { try pool.ensureUnusedCapacity(allocator, 1); const extra_index = try pool.addHashedExtra( @@ -1801,7 +1893,7 @@ pub const Pool = struct { if (kind.isForward()) return if (ty.hasRuntimeBitsIgnoreComptime(zcu)) fwd_decl else - CType.void; + .void; const loaded_tag = loaded_union.loadTagType(ip); const scratch_top = scratch.items.len; defer scratch.shrinkRetainingCapacity(scratch_top); @@ -1848,7 +1940,7 @@ pub const Pool = struct { @typeInfo(Field).@"struct".fields.len, )); if (!has_tag) { - if (fields_len == 0) return CType.void; + if (fields_len == 0) return .void; try pool.ensureUnusedCapacity(allocator, 1); const extra_index = try pool.addHashedExtra( allocator, @@ -1915,7 +2007,7 @@ pub const Pool = struct { struct_fields_len += 1; } } - if (struct_fields_len == 0) return CType.void; + if (struct_fields_len == 0) return .void; sortFields(struct_fields[0..struct_fields_len]); return pool.getAggregate(allocator, .{ .tag = .@"struct", @@ -1929,7 +2021,7 @@ pub const Pool = struct { }, mod, kind), } }, - .opaque_type => return CType.void, + .opaque_type => return .void, .enum_type => return pool.fromType( allocator, scratch, @@ -1938,7 +2030,7 @@ pub const Pool = struct { mod, kind, ), - .func_type => |func_info| if (func_info.is_generic) return CType.void else { + .func_type => |func_info| if (func_info.is_generic) return .void else { const scratch_top = scratch.items.len; defer scratch.shrinkRetainingCapacity(scratch_top); try scratch.ensureUnusedCapacity(allocator, func_info.param_types.len); @@ -1952,7 +2044,7 @@ pub const Pool = struct { pt, mod, kind.asParameter(), - ) else CType.void; + ) else .void; for (0..func_info.param_types.len) |param_index| { const param_type = Type.fromInterned( func_info.param_types.get(ip)[param_index], @@ -2033,7 +2125,7 @@ pub const Pool = struct { pub fn eql(map_adapter: @This(), _: CType, _: void, pool_index: usize) bool { return map_adapter.source_info.eqlAdapted( map_adapter.source_pool, - CType.fromPoolIndex(pool_index), + .fromPoolIndex(pool_index), map_adapter.pool, map_adapter.pool_adapter, ); @@ -2047,7 +2139,7 @@ pub const Pool = struct { .pool_adapter = pool_adapter, }); errdefer _ = pool.map.pop(); - const ctype = CType.fromPoolIndex(gop.index); + const ctype: CType = .fromPoolIndex(gop.index); if (!gop.found_existing) switch (source_info) { .basic => unreachable, .pointer => |pointer_info| pool.items.appendAssumeCapacity(.{ @@ -2232,7 +2324,7 @@ pub const Pool = struct { CTypeAdapter{ .pool = pool }, ); if (!gop.found_existing) pool.items.appendAssumeCapacity(.{ .tag = tag, .data = data }); - return CType.fromPoolIndex(gop.index); + return .fromPoolIndex(gop.index); } fn tagExtra( @@ -2290,7 +2382,7 @@ pub const Pool = struct { pool.extra.shrinkRetainingCapacity(extra_index) else pool.items.appendAssumeCapacity(.{ .tag = tag, .data = extra_index }); - return CType.fromPoolIndex(gop.index); + return .fromPoolIndex(gop.index); } fn sortFields(fields: []Info.Field) void { diff --git a/test/behavior/abs.zig b/test/behavior/abs.zig index bf03237c64..67d3c769d9 100644 --- a/test/behavior/abs.zig +++ b/test/behavior/abs.zig @@ -335,12 +335,12 @@ fn testAbsUnsignedIntVectors(comptime len: comptime_int) !void { test "@abs float vectors" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/12827 if (builtin.zig_backend == .stage2_llvm and diff --git a/test/behavior/x86_64/build.zig b/test/behavior/x86_64/build.zig index 227c3df9f8..6d7a5573e4 100644 --- a/test/behavior/x86_64/build.zig +++ b/test/behavior/x86_64/build.zig @@ -32,6 +32,7 @@ pub fn build(b: *std.Build) void { .cpu_features_sub = std.Target.x86.featureSet(&.{ .cmov, //.sse, + .sse2, }), }, //.{ diff --git a/test/behavior/x86_64/math.zig b/test/behavior/x86_64/math.zig index ed33e9efd0..9547887d38 100644 --- a/test/behavior/x86_64/math.zig +++ b/test/behavior/x86_64/math.zig @@ -22,16 +22,20 @@ inline fn sign(rhs: anytype) switch (@typeInfo(@TypeOf(rhs))) { .vector => |vector| @Vector(vector.len, bool), } { switch (@typeInfo(@TypeOf(rhs))) { - else => return @as(@Type(.{ .int = .{ - .signedness = .signed, - .bits = @bitSizeOf(@TypeOf(rhs)), - } }), @bitCast(rhs)) < 0, + else => { + const I = @Type(.{ .int = .{ + .signedness = .unsigned, + .bits = @bitSizeOf(@TypeOf(rhs)), + } }); + return @as(I, @bitCast(rhs)) & @as(I, 1) << (@bitSizeOf(I) - 1) != 0; + }, .vector => |vector| { - const V = @Vector(vector.len, @Type(.{ .int = .{ - .signedness = .signed, + const I = @Type(.{ .int = .{ + .signedness = .unsigned, .bits = @bitSizeOf(vector.child), - } })); - return @as(V, @bitCast(rhs)) < @as(V, @splat(0)); + } }); + const V = @Vector(vector.len, I); + return @as(V, @bitCast(rhs)) & @as(V, @splat(@as(I, 1) << (@bitSizeOf(I) - 1))) != @as(V, @splat(0)); }, } } @@ -40,7 +44,7 @@ inline fn boolAnd(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) { .bool => return lhs and rhs, .vector => |vector| switch (vector.child) { bool => { - const Bits = @Vector(vector.len, u1); + const Bits = @Type(.{ .int = .{ .signedness = .unsigned, .bits = vector.len } }); const lhs_bits: Bits = @bitCast(lhs); const rhs_bits: Bits = @bitCast(rhs); return @bitCast(lhs_bits & rhs_bits); @@ -56,7 +60,7 @@ inline fn boolOr(lhs: anytype, rhs: @TypeOf(lhs)) @TypeOf(lhs) { .bool => return lhs or rhs, .vector => |vector| switch (vector.child) { bool => { - const Bits = @Vector(vector.len, u1); + const Bits = @Type(.{ .int = .{ .signedness = .unsigned, .bits = vector.len } }); const lhs_bits: Bits = @bitCast(lhs); const rhs_bits: Bits = @bitCast(rhs); return @bitCast(lhs_bits | rhs_bits); @@ -1179,6 +1183,145 @@ fn Unary(comptime op: anytype) type { try testArgs(@Vector(3, i1025), .{ -1 << 1024, -1, 0 }); try testArgs(@Vector(3, u1025), .{ 0, 1, 1 << 1024 }); } + fn testFloatVectorTypes() !void { + try testArgs(@Vector(1, f16), .{ + -0x1.17cp-12, + }); + try testArgs(@Vector(2, f16), .{ + 0x1.47cp9, 0x1.3acp9, + }); + try testArgs(@Vector(4, f16), .{ + 0x1.ab4p0, -0x1.7fcp-7, -0x1.1cp0, -0x1.f14p12, + }); + try testArgs(@Vector(8, f16), .{ + -0x1.8d8p8, 0x1.83p10, -0x1.5ap-1, -0x1.d78p13, -0x1.608p12, 0x1.e8p-9, -0x1.688p-10, -0x1.738p9, + }); + try testArgs(@Vector(16, f16), .{ + 0x1.da8p-1, -0x1.ed4p-10, -0x1.dc8p1, 0x1.b78p-14, nan(f16), 0x1.9d8p8, nan(f16), 0x1.d5p13, + -0x1.2dp13, 0x1.6c4p12, 0x1.a9cp-11, -0x1.0ecp8, 0x0.4ccp-14, -0x1.0a8p-6, -0x1.5bcp-14, 0x1.6d8p-9, + }); + try testArgs(@Vector(32, f16), .{ + 0x1.d5cp-6, -0x1.a98p5, 0x1.49cp5, -0x1.e4p-1, -0x1.21p-13, -0x1.c94p-1, -0x1.adcp-5, -0x1.524p-1, + -0x1.0d8p-3, -0x1.5c4p-2, 0x1.f84p-2, 0x1.664p1, -0x1.f64p13, -0x1.bf4p4, -0x1.4b8p0, -0x0.f64p-14, + -0x1.3f8p1, 0x1.098p2, -0x1.a44p8, 0x1.048p13, 0x1.fd4p-11, 0x1.18p-9, -0x1.504p2, 0x1.d04p7, + -nan(f16), 0x1.a94p2, 0x0.5e8p-14, -0x1.7acp-7, 0x1.4c8p-3, 0x1.518p-4, nan(f16), 0x1.8f8p10, + }); + try testArgs(@Vector(64, f16), .{ + -0x1.c2p2, 0x0.2fcp-14, 0x1.de8p0, -0x1.714p2, 0x1.f9p-7, -0x1.11cp-13, -0x1.558p10, -0x1.2acp-7, + 0x1.348p14, 0x1.2dcp7, -0x1.8acp-12, -0x1.2cp2, 0x1.868p1, -0x1.1f8p-14, 0x1.638p7, -0x1.734p-5, + 0x0.b98p-14, -0x1.7f4p-12, -0x1.38cp15, 0x1.50cp15, 0x1.91cp8, 0x1.cb4p-1, 0x1.fc4p-13, 0x1.9a4p0, + 0x1.18p-4, 0x1.60cp10, 0x1.6fp-12, 0x1.b48p6, 0x1.37cp-11, 0x1.424p7, 0x1.44cp13, 0x1.aep5, + 0x1.968p14, 0x1.e8p13, -0x1.bp2, -0x1.644p5, 0x1.de4p-8, -0x1.5b4p-14, -0x1.4ap1, -0x1.868p9, + -0x1.d14p0, 0x1.d7cp15, 0x1.3c8p14, 0x1.2ccp-14, -0x1.ee4p8, 0x1.49p-3, 0x1.35cp12, 0x1.d34p6, + 0x1.7acp3, -0x1.fa4p2, 0x1.7b4p13, -0x1.cf4p-12, -0x1.ebcp-10, -0x1.5p-3, 0x1.4bp-6, 0x1.83p12, + -0x1.f9cp-8, -0x1.43p-8, -0x1.99p-1, -0x1.dacp3, -0x1.728p-4, -0x1.03cp4, 0x1.604p-2, -0x1.0ep13, + }); + + try testArgs(@Vector(1, f32), .{ + -0x1.17cp-12, + }); + try testArgs(@Vector(2, f32), .{ + -0x1.a3123ap90, -0x1.4a2ec6p-54, + }); + try testArgs(@Vector(4, f32), .{ + -0x1.8a41p77, -0x1.7c54e2p-61, -0x1.498556p-41, 0x1.d77c22p-20, + }); + try testArgs(@Vector(8, f32), .{ + 0x1.943da4p-86, 0x1.528792p95, -0x1.9c9bfap-26, -0x1.8df936p-90, + -0x1.6a70cep56, 0x1.626638p-48, 0x1.7bb2bap-57, -0x1.ac5104p94, + }); + try testArgs(@Vector(16, f32), .{ + 0x1.157044p115, -0x1.416c04p-111, 0x1.a8f164p-104, 0x1.9b6678p84, + -0x1.9d065cp9, -0x1.e8c4b4p126, -0x1.ddb968p84, -0x1.fec8c8p74, + 0x1.64ffb2p59, 0x1.548922p20, 0x1.7270fcp22, -0x1.abac68p33, + 0x1.faabfp33, -0x1.8aee82p55, 0x1.1bf8fp75, 0x1.33c46ap-66, + }); + try testArgs(@Vector(32, f32), .{ + -0x1.039b68p37, -0x1.34de4ap-74, -0x1.05d78ap-76, -0x1.be0f5ap-47, + 0x1.032204p-38, 0x1.ef8e2ap-78, -0x1.b013ecp-80, 0x1.71fe4cp99, + 0x1.abdadap-14, 0x1.56a9a8p-48, -0x1.8bbd7ep9, 0x1.edd308p-72, + -0x1.92fafcp-121, -0x1.50812p19, 0x1.f4ddc4p28, -0x1.6f0b12p-50, + -0x1.12ab02p127, 0x1.24df48p21, -0x1.993c3p-14, -0x1.4cc476p-112, + 0x1.13d9a8p-40, 0x1.a6e652p-9, -0x1.9c730cp-21, -0x1.a75aaap-70, + -0x1.39e632p-111, 0x1.8e8da8p-45, 0x1.b5652cp31, 0x1.258366p44, + 0x1.d473aap92, -0x1.951b64p9, 0x1.542edp15, -0x0.f6222ap-126, + }); + + try testArgs(@Vector(1, f64), .{ + -0x1.0114613df6f97p816, + }); + try testArgs(@Vector(2, f64), .{ + -0x1.8404dad72003cp720, -0x1.6b14b40bcf3b7p-176, + }); + try testArgs(@Vector(4, f64), .{ + -0x1.04e1acbfddd9cp681, -0x1.ed553cc056da7p-749, + 0x1.3d3f703a0c893p-905, 0x1.0b35633fa78fp691, + }); + try testArgs(@Vector(8, f64), .{ + -0x1.901a2a60f0562p-301, -0x1.2516175ad61ecp-447, + 0x1.e7b12124846bfp564, 0x1.9291384bd7259p209, + -0x1.a7bf62f803c98p900, 0x1.4e2e26257bb3p987, + -0x1.413ca9a32d894p811, 0x1.61b1dd9432e95p479, + }); + try testArgs(@Vector(16, f64), .{ + -0x1.8fc7286d95f54p-235, -0x1.796a7ea8372b6p-837, + -0x1.8c0f930539acbp-98, -0x1.ec80dfbf0b931p-430, + -0x1.e3d80c640652fp-1019, 0x1.8241238fb542fp161, + -0x1.e1f1a79d50263p137, -0x1.9ac5cb2771c28p-791, + 0x1.4d8f00fe881e7p-401, -0x1.87fbd7bfd99d7p346, + -0x1.a8a7cc575335ep1017, 0x1.37bb88dc3fd8bp-355, + 0x1.9d53d346c0e65p929, -0x1.bbae3d0229c34p289, + -0x1.cb8ef994d5ce5p25, 0x1.ba20af512616ap50, + }); + + try testArgs(@Vector(1, f80), .{ + -0x1.a2e9410a7dfedabp-2324, + }); + try testArgs(@Vector(2, f80), .{ + -0x1.a2e9410a7dfedabp-2324, + 0x1.2b17da3b9746885p-8665, + }); + try testArgs(@Vector(4, f80), .{ + -0x1.c488fedb7ab646cep-13007, + 0x1.e914deaccaa50016p2073, + -0x1.d1c7ae8ec3c9df86p10642, + -0x1.2da1658f337fa01p9893, + }); + try testArgs(@Vector(8, f80), .{ + -0x1.bed8a74c43750656p890, + -0x1.7bf57f38004ac976p8481, + -0x1.9cdc10ac0657d328p7884, + 0x1.c86f61883da149fp12293, + -0x1.528d6957df6bfdd8p14125, + -0x1.5ebb4006d0243bfep14530, + -0x1.94b9b18636d12402p-1845, + -0x1.25439a6d68add188p5962, + }); + + try testArgs(@Vector(1, f128), .{ + -0x1.d1e6fc3b1e66632e7b79051a47dap14300, + }); + try testArgs(@Vector(2, f128), .{ + 0x1.84b3ac8ffe5893b2c6af8d68de9dp-83, + -0x1.438ca2c8a0d8e3ee9062d351c46ep-10235, + }); + try testArgs(@Vector(4, f128), .{ + 0x1.04eb03882d4fd1b090e714d3e5ep806, + -0x1.4082b29f7c26e701764c915642ffp-6182, + -0x1.b6f1e8565e5040415110f18b519ap13383, + 0x1.1c29f8c162cead9061c5797ea15ap11957, + }); + try testArgs(@Vector(8, f128), .{ + -0x1.53d7f00cd204d80e5ff5bb665773p11218, + -0x1.4daa1c81cffe28e8fa5cd703c287p2362, + -0x1.cc6a71c3ad4560871efdbd025cd7p-8116, + -0x1.87f8553cf8772fb6b78e7df3e3bap14523, + -0x1.14b6880f6678f86dfb543dde1c6ep2105, + 0x1.9d2d4398414da9d857e76e8fd7ccp-13668, + 0x1.a37f07af240ded458d103c022064p-1158, + 0x1.425d53e6bd6070b847e5da1ed593p1394, + }); + } }; } @@ -1601,6 +1744,7 @@ test abs { try Unary(abs).testIntTypes(); try Unary(abs).testIntVectorTypes(); try Unary(abs).testFloatTypes(); + try Unary(abs).testFloatVectorTypes(); } inline fn clz(comptime Type: type, rhs: Type) @TypeOf(@clz(rhs)) { diff --git a/test/src/Debugger.zig b/test/src/Debugger.zig index d14ea8ecd5..f8948eb7b9 100644 --- a/test/src/Debugger.zig +++ b/test/src/Debugger.zig @@ -526,8 +526,9 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\ var null_u32: ?u32 = null; \\ var maybe_u32: ?u32 = null; \\ var nonnull_u32: ?u32 = 456; + \\ null_u32 = null_u32; \\ maybe_u32 = 123; - \\ _ = .{ &null_u32, &nonnull_u32 }; + \\ nonnull_u32 = nonnull_u32; \\ } \\} \\ @@ -539,7 +540,7 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\frame variable -- null_u32 maybe_u32 nonnull_u32 \\breakpoint delete --force 1 \\ - \\breakpoint set --file optionals.zig --source-pattern-regexp '_ = \.{ &null_u32, &nonnull_u32 };' + \\breakpoint set --file optionals.zig --source-pattern-regexp 'nonnull_u32 = nonnull_u32;' \\process continue \\frame variable --show-types -- null_u32 maybe_u32 nonnull_u32 \\breakpoint delete --force 2 @@ -1285,10 +1286,12 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\ mod1.m1cfi(r0pai ^ 8); \\} \\pub fn r0cf(r0ca: u32) void { - \\ _ = r0ca; + \\ var discard = r0ca; + \\ _ = &discard; \\} \\pub inline fn r0cfi(r0cai: u32) void { - \\ _ = r0cai; + \\ var discard = r0cai; + \\ _ = &discard; \\} \\pub fn main() void { \\ root0.r0pf(12); @@ -1331,10 +1334,12 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\ mod1.m1cfi(r1pai ^ 8); \\} \\pub fn r1cf(r1ca: u32) void { - \\ _ = r1ca; + \\ var discard = r1ca; + \\ _ = &discard; \\} \\pub inline fn r1cfi(r1cai: u32) void { - \\ _ = r1cai; + \\ var discard = r1cai; + \\ _ = &discard; \\} \\ , @@ -1368,10 +1373,12 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\ mod1.m1cfi(m0pai ^ 8); \\} \\pub fn m0cf(m0ca: u32) void { - \\ _ = m0ca; + \\ var discard = m0ca; + \\ _ = &discard; \\} \\pub inline fn m0cfi(m0cai: u32) void { - \\ _ = m0cai; + \\ var discard = m0cai; + \\ _ = &discard; \\} \\ , @@ -1404,10 +1411,12 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\ mod1.m1cfi(m1pai ^ 8); \\} \\pub fn m1cf(m1ca: u32) void { - \\ _ = m1ca; + \\ var discard = m1ca; + \\ _ = &discard; \\} \\pub inline fn m1cfi(m1cai: u32) void { - \\ _ = m1cai; + \\ var discard = m1cai; + \\ _ = &discard; \\} \\ , @@ -1416,13 +1425,13 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { \\settings set frame-format 'frame #${frame.index}:{ ${module.file.basename}{\`${function.name-with-args}{${frame.no-debug}${function.pc-offset}}}}{ at ${line.file.basename}:${line.number}{:${line.column}}}{${function.is-optimized} [opt]}{${frame.is-artificial} [artificial]}\n' \\ \\breakpoint set --file root0.zig --line 26 - \\breakpoint set --file root0.zig --line 29 + \\breakpoint set --file root0.zig --line 30 \\breakpoint set --file root1.zig --line 26 - \\breakpoint set --file root1.zig --line 29 + \\breakpoint set --file root1.zig --line 30 \\breakpoint set --file mod0.zig --line 26 - \\breakpoint set --file mod0.zig --line 29 + \\breakpoint set --file mod0.zig --line 30 \\breakpoint set --file mod1.zig --line 26 - \\breakpoint set --file mod1.zig --line 29 + \\breakpoint set --file mod1.zig --line 30 \\ \\process launch \\thread backtrace --count 3 @@ -1563,259 +1572,259 @@ pub fn addTestsForTarget(db: *Debugger, target: Target) void { &.{ \\ * frame #0: inline_call`root0.r0cf(r0ca=13) at root0.zig:26:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:6:15 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , - \\ * frame #0: inline_call`root0.r0pf [inlined] r0cfi(r0cai=14) at root0.zig:29:5 + \\ * frame #0: inline_call`root0.r0pf [inlined] r0cfi(r0cai=14) at root0.zig:30:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:7:16 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , \\ * frame #0: inline_call`root1.r1cf(r1ca=15) at root1.zig:26:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:8:15 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , - \\ * frame #0: inline_call`root0.r0pf [inlined] r1cfi(r1cai=8) at root1.zig:29:5 + \\ * frame #0: inline_call`root0.r0pf [inlined] r1cfi(r1cai=8) at root1.zig:30:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:9:16 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=9) at mod0.zig:26:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:10:14 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , - \\ * frame #0: inline_call`root0.r0pf [inlined] m0cfi(m0cai=10) at mod0.zig:29:5 + \\ * frame #0: inline_call`root0.r0pf [inlined] m0cfi(m0cai=10) at mod0.zig:30:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:11:15 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=11) at mod1.zig:26:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:12:14 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , - \\ * frame #0: inline_call`root0.r0pf [inlined] m1cfi(m1cai=4) at mod1.zig:29:5 + \\ * frame #0: inline_call`root0.r0pf [inlined] m1cfi(m1cai=4) at mod1.zig:30:5 \\ frame #1: inline_call`root0.r0pf(r0pa=12) at root0.zig:13:15 - \\ frame #2: inline_call`root0.main at root0.zig:32:15 + \\ frame #2: inline_call`root0.main at root0.zig:34:15 , \\ * frame #0: inline_call`root0.r0cf(r0ca=22) at root0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:16:15 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , - \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=21) at root0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=21) at root0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:17:16 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , \\ * frame #0: inline_call`root1.r1cf(r1ca=20) at root1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:18:15 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , - \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=19) at root1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=19) at root1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:19:16 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=18) at mod0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:20:14 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , - \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=17) at mod0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=17) at mod0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:21:15 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=16) at mod1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:22:14 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , - \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=31) at mod1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=31) at mod1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r0pfi(r0pai=23) at root0.zig:23:15 - \\ frame #2: inline_call`root0.main at root0.zig:33:16 + \\ frame #2: inline_call`root0.main at root0.zig:35:16 , \\ * frame #0: inline_call`root0.r0cf(r0ca=35) at root0.zig:26:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:6:15 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , - \\ * frame #0: inline_call`root1.r1pf [inlined] r0cfi(r0cai=32) at root0.zig:29:5 + \\ * frame #0: inline_call`root1.r1pf [inlined] r0cfi(r0cai=32) at root0.zig:30:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:7:16 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , \\ * frame #0: inline_call`root1.r1cf(r1ca=33) at root1.zig:26:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:8:15 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , - \\ * frame #0: inline_call`root1.r1pf [inlined] r1cfi(r1cai=38) at root1.zig:29:5 + \\ * frame #0: inline_call`root1.r1pf [inlined] r1cfi(r1cai=38) at root1.zig:30:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:9:16 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=39) at mod0.zig:26:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:10:14 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , - \\ * frame #0: inline_call`root1.r1pf [inlined] m0cfi(m0cai=36) at mod0.zig:29:5 + \\ * frame #0: inline_call`root1.r1pf [inlined] m0cfi(m0cai=36) at mod0.zig:30:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:11:15 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=37) at mod1.zig:26:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:12:14 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , - \\ * frame #0: inline_call`root1.r1pf [inlined] m1cfi(m1cai=42) at mod1.zig:29:5 + \\ * frame #0: inline_call`root1.r1pf [inlined] m1cfi(m1cai=42) at mod1.zig:30:5 \\ frame #1: inline_call`root1.r1pf(r1pa=34) at root1.zig:13:15 - \\ frame #2: inline_call`root0.main at root0.zig:34:15 + \\ frame #2: inline_call`root0.main at root0.zig:36:15 , \\ * frame #0: inline_call`root0.r0cf(r0ca=44) at root0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:16:15 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , - \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=47) at root0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=47) at root0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:17:16 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , \\ * frame #0: inline_call`root1.r1cf(r1ca=46) at root1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:18:15 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , - \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=41) at root1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=41) at root1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:19:16 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=40) at mod0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:20:14 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , - \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=43) at mod0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=43) at mod0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:21:15 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=42) at mod1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:22:14 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , - \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=37) at mod1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=37) at mod1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] r1pfi(r1pai=45) at root1.zig:23:15 - \\ frame #2: inline_call`root0.main at root0.zig:35:16 + \\ frame #2: inline_call`root0.main at root0.zig:37:16 , \\ * frame #0: inline_call`root0.r0cf(r0ca=57) at root0.zig:26:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:6:15 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , - \\ * frame #0: inline_call`mod0.m0pf [inlined] r0cfi(r0cai=58) at root0.zig:29:5 + \\ * frame #0: inline_call`mod0.m0pf [inlined] r0cfi(r0cai=58) at root0.zig:30:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:7:16 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , \\ * frame #0: inline_call`root1.r1cf(r1ca=59) at root1.zig:26:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:8:15 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , - \\ * frame #0: inline_call`mod0.m0pf [inlined] r1cfi(r1cai=60) at root1.zig:29:5 + \\ * frame #0: inline_call`mod0.m0pf [inlined] r1cfi(r1cai=60) at root1.zig:30:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:9:16 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=61) at mod0.zig:26:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:10:14 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , - \\ * frame #0: inline_call`mod0.m0pf [inlined] m0cfi(m0cai=62) at mod0.zig:29:5 + \\ * frame #0: inline_call`mod0.m0pf [inlined] m0cfi(m0cai=62) at mod0.zig:30:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:11:15 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=63) at mod1.zig:26:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:12:14 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , - \\ * frame #0: inline_call`mod0.m0pf [inlined] m1cfi(m1cai=48) at mod1.zig:29:5 + \\ * frame #0: inline_call`mod0.m0pf [inlined] m1cfi(m1cai=48) at mod1.zig:30:5 \\ frame #1: inline_call`mod0.m0pf(m0pa=56) at mod0.zig:13:15 - \\ frame #2: inline_call`root0.main at root0.zig:36:14 + \\ frame #2: inline_call`root0.main at root0.zig:38:14 , \\ * frame #0: inline_call`root0.r0cf(r0ca=66) at root0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:16:15 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , - \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=65) at root0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=65) at root0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:17:16 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , \\ * frame #0: inline_call`root1.r1cf(r1ca=64) at root1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:18:15 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , - \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=71) at root1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=71) at root1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:19:16 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=70) at mod0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:20:14 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , - \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=69) at mod0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=69) at mod0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:21:15 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=68) at mod1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:22:14 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , - \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=75) at mod1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=75) at mod1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m0pfi(m0pai=67) at mod0.zig:23:15 - \\ frame #2: inline_call`root0.main at root0.zig:37:15 + \\ frame #2: inline_call`root0.main at root0.zig:39:15 , \\ * frame #0: inline_call`root0.r0cf(r0ca=79) at root0.zig:26:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:6:15 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , - \\ * frame #0: inline_call`mod1.m1pf [inlined] r0cfi(r0cai=76) at root0.zig:29:5 + \\ * frame #0: inline_call`mod1.m1pf [inlined] r0cfi(r0cai=76) at root0.zig:30:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:7:16 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , \\ * frame #0: inline_call`root1.r1cf(r1ca=77) at root1.zig:26:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:8:15 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , - \\ * frame #0: inline_call`mod1.m1pf [inlined] r1cfi(r1cai=74) at root1.zig:29:5 + \\ * frame #0: inline_call`mod1.m1pf [inlined] r1cfi(r1cai=74) at root1.zig:30:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:9:16 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=75) at mod0.zig:26:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:10:14 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , - \\ * frame #0: inline_call`mod1.m1pf [inlined] m0cfi(m0cai=72) at mod0.zig:29:5 + \\ * frame #0: inline_call`mod1.m1pf [inlined] m0cfi(m0cai=72) at mod0.zig:30:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:11:15 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=73) at mod1.zig:26:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:12:14 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , - \\ * frame #0: inline_call`mod1.m1pf [inlined] m1cfi(m1cai=70) at mod1.zig:29:5 + \\ * frame #0: inline_call`mod1.m1pf [inlined] m1cfi(m1cai=70) at mod1.zig:30:5 \\ frame #1: inline_call`mod1.m1pf(m1pa=78) at mod1.zig:13:15 - \\ frame #2: inline_call`root0.main at root0.zig:38:14 + \\ frame #2: inline_call`root0.main at root0.zig:40:14 , \\ * frame #0: inline_call`root0.r0cf(r0ca=88) at root0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:16:15 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , - \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=91) at root0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r0cfi(r0cai=91) at root0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:17:16 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , \\ * frame #0: inline_call`root1.r1cf(r1ca=90) at root1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:18:15 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , - \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=93) at root1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] r1cfi(r1cai=93) at root1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:19:16 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , \\ * frame #0: inline_call`mod0.m0cf(m0ca=92) at mod0.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:20:14 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , - \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=95) at mod0.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m0cfi(m0cai=95) at mod0.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:21:15 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , \\ * frame #0: inline_call`mod1.m1cf(m1ca=94) at mod1.zig:26:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:22:14 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 , - \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=81) at mod1.zig:29:5 + \\ * frame #0: inline_call`root0.main [inlined] m1cfi(m1cai=81) at mod1.zig:30:5 \\ frame #1: inline_call`root0.main [inlined] m1pfi(m1pai=89) at mod1.zig:23:15 - \\ frame #2: inline_call`root0.main at root0.zig:39:15 + \\ frame #2: inline_call`root0.main at root0.zig:41:15 }, ); db.addLldbTest( From ba82d6e83e3e0dc00ad235fae52c21f9014ebd78 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Thu, 23 Jan 2025 06:58:40 -0500 Subject: [PATCH 3/5] x86_64: fix typo and lower optimized insts --- src/arch/x86_64/CodeGen.zig | 59 ++++++++++--------- test/behavior/floatop.zig | 1 + .../anytype_param_requires_comptime.zig | 2 +- .../bogus_method_call_on_slice.zig | 2 +- .../compile_errors/coerce_anon_struct.zig | 2 +- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 734b863d7e..7360fae0b3 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2440,10 +2440,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .shr, .shr_exact => try cg.airShlShrBinOp(inst), .shl, .shl_exact => try cg.airShlShrBinOp(inst), - .mul => try cg.airMulDivBinOp(inst), - .mul_wrap => try cg.airMulDivBinOp(inst), - .rem => try cg.airMulDivBinOp(inst), - .mod => try cg.airMulDivBinOp(inst), + .mul, + .mul_wrap, + .rem, + .mod, + .div_float, + .div_trunc, + .div_floor, + .div_exact, + => |air_tag| try cg.airMulDivBinOp(inst, air_tag), .add_sat => try cg.airAddSat(inst), .sub_sat => try cg.airSubSat(inst), @@ -2465,15 +2470,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .ceil => try cg.airRound(inst, .{ .mode = .up, .precision = .inexact }), .trunc_float => try cg.airRound(inst, .{ .mode = .zero, .precision = .inexact }), .sqrt => try cg.airSqrt(inst), - .neg => try cg.airFloatSign(inst), + .neg => |air_tag| try cg.airFloatSign(inst, air_tag), .add_with_overflow => try cg.airAddSubWithOverflow(inst), .sub_with_overflow => try cg.airAddSubWithOverflow(inst), .mul_with_overflow => try cg.airMulWithOverflow(inst), .shl_with_overflow => try cg.airShlWithOverflow(inst), - .div_float, .div_trunc, .div_floor, .div_exact => try cg.airMulDivBinOp(inst), - .cmp_lt_errors_len => try cg.airCmpLtErrorsLen(inst), .bitcast => try cg.airBitCast(inst), @@ -2528,19 +2531,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .sub_safe, .mul_safe, => return cg.fail("TODO implement safety_checked_instructions", .{}), - .add_optimized, - .sub_optimized, - .mul_optimized, - .div_float_optimized, - .div_trunc_optimized, - .div_floor_optimized, - .div_exact_optimized, - .rem_optimized, - .mod_optimized, - .neg_optimized, - .reduce_optimized, - .int_from_float_optimized, - => return cg.fail("TODO implement optimized float mode", .{}), + + .add_optimized => try cg.airBinOp(inst, .add), + .sub_optimized => try cg.airBinOp(inst, .sub), + .mul_optimized => try cg.airBinOp(inst, .mul), + .div_float_optimized => try cg.airMulDivBinOp(inst, .div_float), + .div_trunc_optimized => try cg.airMulDivBinOp(inst, .div_trunc), + .div_floor_optimized => try cg.airMulDivBinOp(inst, .div_floor), + .div_exact_optimized => try cg.airMulDivBinOp(inst, .div_exact), + .rem_optimized => try cg.airMulDivBinOp(inst, .rem), + .mod_optimized => try cg.airMulDivBinOp(inst, .mod), + .neg_optimized => try cg.airFloatSign(inst, .neg), + .reduce_optimized => try cg.airReduce(inst), + .int_from_float_optimized => try cg.airIntFromFloat(inst), .arg => try cg.airDbgArg(inst), .ptr_add => |air_tag| if (use_old) try cg.airPtrArithmetic(inst, air_tag) else { @@ -16257,12 +16260,11 @@ fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 { return dst_info.bits; } -fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { +fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const result = result: { - const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; const dst_ty = self.typeOfIndex(inst); switch (dst_ty.zigTypeTag(zcu)) { .float, .vector => break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs), @@ -19600,10 +19602,9 @@ fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void { return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none }); } -fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void { +fn floatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag, operand: Air.Inst.Ref, ty: Type) !void { const pt = self.pt; const zcu = pt.zcu; - const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)]; const result = result: { const scalar_bits = ty.scalarType(zcu).floatBits(self.target.*); @@ -19728,10 +19729,10 @@ fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Ty return self.finishAir(inst, result, .{ operand, .none, .none }); } -fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index) !void { +fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const ty = self.typeOf(un_op); - return self.floatSign(inst, un_op, ty); + return self.floatSign(inst, tag, un_op, ty); } const RoundMode = packed struct(u5) { @@ -20014,7 +20015,7 @@ fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { break :result dst_mcv; }, }, - .float => return self.floatSign(inst, ty_op.operand, ty), + .float => return self.floatSign(inst, .abs, ty_op.operand, ty), .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { else => null, .int => switch (ty.childType(zcu).intInfo(zcu).bits) { @@ -20050,7 +20051,7 @@ fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { 5...8 => if (self.hasFeature(.avx2)) .{ .vp_d, .abs } else null, }, }, - .float => return self.floatSign(inst, ty_op.operand, ty), + .float => return self.floatSign(inst, .abs, ty_op.operand, ty), }, }) orelse return self.fail("TODO implement airAbs for {}", .{ty.fmt(pt)}); @@ -22911,7 +22912,7 @@ fn genBinOp( .mul => .{ .v_ss, .mul }, .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, .max => .{ .v_ss, .max }, - .min => .{ .v_ss, .max }, + .min => .{ .v_ss, .min }, else => unreachable, }, dst_reg, diff --git a/test/behavior/floatop.zig b/test/behavior/floatop.zig index 72b773b76c..d7cf657b90 100644 --- a/test/behavior/floatop.zig +++ b/test/behavior/floatop.zig @@ -1654,6 +1654,7 @@ test "runtime isNan(inf * 0)" { } test "optimized float mode" { + if (builtin.zig_backend != .stage2_llvm) return error.SkipZigTest; if (builtin.mode == .Debug) return error.SkipZigTest; const big = 0x1p40; diff --git a/test/cases/compile_errors/anytype_param_requires_comptime.zig b/test/cases/compile_errors/anytype_param_requires_comptime.zig index 268211ebc5..e10511a2f9 100644 --- a/test/cases/compile_errors/anytype_param_requires_comptime.zig +++ b/test/cases/compile_errors/anytype_param_requires_comptime.zig @@ -15,6 +15,6 @@ pub export fn entry() void { // error // // :7:25: error: unable to resolve comptime value -// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_441.C' must be comptime-known +// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_447.C' must be comptime-known // :4:16: note: struct requires comptime because of this field // :4:16: note: types are not available at runtime diff --git a/test/cases/compile_errors/bogus_method_call_on_slice.zig b/test/cases/compile_errors/bogus_method_call_on_slice.zig index 1308b184bd..51524416c0 100644 --- a/test/cases/compile_errors/bogus_method_call_on_slice.zig +++ b/test/cases/compile_errors/bogus_method_call_on_slice.zig @@ -16,5 +16,5 @@ pub export fn entry2() void { // // :3:6: error: no field or member function named 'copy' in '[]const u8' // :9:8: error: no field or member function named 'bar' in '@TypeOf(.{})' -// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_445' +// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_451' // :12:6: note: struct declared here diff --git a/test/cases/compile_errors/coerce_anon_struct.zig b/test/cases/compile_errors/coerce_anon_struct.zig index c25cf0bea8..c043e190e7 100644 --- a/test/cases/compile_errors/coerce_anon_struct.zig +++ b/test/cases/compile_errors/coerce_anon_struct.zig @@ -6,6 +6,6 @@ export fn foo() void { // error // -// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_434' +// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_440' // :3:16: note: struct declared here // :1:11: note: struct declared here From 7701cfa032a0d3611310426d5a5027d8862d49b1 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Thu, 23 Jan 2025 08:07:37 -0500 Subject: [PATCH 4/5] x86_64: mitigate miscomp during switch dispatch --- src/arch/x86_64/CodeGen.zig | 97 ++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 7360fae0b3..85a3c009d5 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -26596,6 +26596,9 @@ fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void { if (switch_br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); } + // Ensure a register is available for dispatch. + if (!mat_cond.isRegister()) _ = try self.register_manager.allocReg(null, abi.RegisterClass.gp); + self.scope_generation += 1; const state = try self.saveState(); @@ -26618,47 +26621,63 @@ fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void { const block_ty = self.typeOfIndex(br.block_inst); const loop_data = self.loops.getPtr(br.block_inst).?; + const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; + { + try self.getValue(block_tracking.short, null); + const src_mcv = try self.resolveInst(br.operand); + + if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { + try self.getValue(block_tracking.short, br.block_inst); + // .long = .none to avoid merging operand and block result stack frames. + const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; + try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*); + for (current_tracking.getRegs()) |src_reg| self.register_manager.freeReg(src_reg); + } else { + try self.getValue(block_tracking.short, br.block_inst); + try self.genCopy(block_ty, block_tracking.short, try self.resolveInst(br.operand), .{}); + } + } + + // Process operand death so that it is properly accounted for in the State below. + if (self.liveness.operandDies(inst, 0)) { + if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); + } + + try self.restoreState(loop_data.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = false, + }); + if (self.loop_switches.getPtr(br.block_inst)) |table| { - // Process operand death so that it is properly accounted for in the State below. - const condition_dies = self.liveness.operandDies(inst, 0); - - try self.restoreState(loop_data.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - const condition_ty = self.typeOf(br.operand); - const condition = try self.resolveInst(br.operand); - const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: { - const condition_index = try self.allocTempRegOrMem(condition_ty, true); - try self.genCopy(condition_ty, condition_index, condition, .{}); - break :condition_index condition_index; - }; + const condition_mcv = block_tracking.short; try self.spillEflagsIfOccupied(); if (table.min.orderAgainstZero(self.pt.zcu).compare(.neq)) try self.genBinOpMir( .{ ._, .sub }, condition_ty, - condition_index, + condition_mcv, .{ .air_ref = Air.internedToRef(table.min.toIntern()) }, ); switch (table.else_relocs) { .@"unreachable" => {}, .forward => |*else_relocs| { - try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table.len - 1 }); + try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_mcv, .{ .immediate = table.len - 1 }); try else_relocs.append(self.gpa, try self.asmJccReloc(.a, undefined)); }, .backward => |else_reloc| { - try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_index, .{ .immediate = table.len - 1 }); + try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition_mcv, .{ .immediate = table.len - 1 }); _ = try self.asmJccReloc(.a, else_reloc); }, } { - const condition_index_reg = if (condition_index.isRegister()) - condition_index.getReg().? - else - try self.copyToTmpRegister(.usize, condition_index); + const condition_index_reg = if (condition_mcv.isRegister()) condition_mcv.getReg().? else cond: { + const condition_index_reg = + RegisterManager.regAtTrackedIndex(@intCast(loop_data.state.free_registers.findFirstSet().?)); + try self.genSetReg(condition_index_reg, condition_ty, condition_mcv, .{}); + break :cond condition_index_reg; + }; const condition_index_lock = self.register_manager.lockReg(condition_index_reg); defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock); try self.truncateRegister(condition_ty, condition_index_reg); @@ -26677,38 +26696,6 @@ fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void { return self.finishAir(inst, .none, .{ br.operand, .none, .none }); } - const block_tracking = self.inst_tracking.getPtr(br.block_inst).?; - done: { - try self.getValue(block_tracking.short, null); - const src_mcv = try self.resolveInst(br.operand); - - if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) { - try self.getValue(block_tracking.short, br.block_inst); - // .long = .none to avoid merging operand and block result stack frames. - const current_tracking: InstTracking = .{ .long = .none, .short = src_mcv }; - try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*); - for (current_tracking.getRegs()) |src_reg| self.register_manager.freeReg(src_reg); - break :done; - } - - try self.getValue(block_tracking.short, br.block_inst); - const dst_mcv = block_tracking.short; - try self.genCopy(block_ty, dst_mcv, try self.resolveInst(br.operand), .{}); - break :done; - } - - // Process operand death so that it is properly accounted for in the State below. - if (self.liveness.operandDies(inst, 0)) { - if (br.operand.toIndex()) |op_inst| try self.processDeath(op_inst); - } - - try self.restoreState(loop_data.state, &.{}, .{ - .emit_instructions = true, - .update_tracking = false, - .resurrect = false, - .close_scope = false, - }); - // Emit a jump with a relocation. It will be patched up after the block ends. // Leave the jump offset undefined _ = try self.asmJmpReloc(loop_data.target); From c7433212d1bf96adbfc6f3d706404436c0c98c20 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Fri, 24 Jan 2025 14:00:15 -0500 Subject: [PATCH 5/5] x86_64: rewrite scalar and vector int `@min` and `@max` --- lib/std/zig/Zir.zig | 18 +- src/Air.zig | 16 + src/InternPool.zig | 80 + src/Sema.zig | 16 + src/Type.zig | 16 + src/arch/x86_64/CodeGen.zig | 5340 ++++++++++++++++- src/codegen/c/Type.zig | 240 + test/behavior/maximum_minimum.zig | 8 +- test/behavior/x86_64/math.zig | 418 +- .../anytype_param_requires_comptime.zig | 2 +- .../bogus_method_call_on_slice.zig | 2 +- .../compile_errors/coerce_anon_struct.zig | 2 +- 12 files changed, 6062 insertions(+), 96 deletions(-) diff --git a/lib/std/zig/Zir.zig b/lib/std/zig/Zir.zig index 2ee4053331..f26bc288f4 100644 --- a/lib/std/zig/Zir.zig +++ b/lib/std/zig/Zir.zig @@ -2128,7 +2128,7 @@ pub const Inst = struct { ref_start_index = static_len, _, - pub const static_len = 76; + pub const static_len = 92; pub fn toRef(i: Index) Inst.Ref { return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i)); @@ -2211,6 +2211,22 @@ pub const Inst = struct { single_const_pointer_to_comptime_int_type, slice_const_u8_type, slice_const_u8_sentinel_0_type, + vector_16_i8_type, + vector_32_i8_type, + vector_16_u8_type, + vector_32_u8_type, + vector_8_i16_type, + vector_16_i16_type, + vector_8_u16_type, + vector_16_u16_type, + vector_4_i32_type, + vector_8_i32_type, + vector_4_u32_type, + vector_8_u32_type, + vector_2_i64_type, + vector_4_i64_type, + vector_2_u64_type, + vector_4_u64_type, vector_4_f16_type, vector_8_f16_type, vector_4_f32_type, diff --git a/src/Air.zig b/src/Air.zig index acc1c7f2e2..98f1ba8c71 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -984,6 +984,22 @@ pub const Inst = struct { single_const_pointer_to_comptime_int_type = @intFromEnum(InternPool.Index.single_const_pointer_to_comptime_int_type), slice_const_u8_type = @intFromEnum(InternPool.Index.slice_const_u8_type), slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type), + vector_16_i8_type = @intFromEnum(InternPool.Index.vector_16_i8_type), + vector_32_i8_type = @intFromEnum(InternPool.Index.vector_32_i8_type), + vector_16_u8_type = @intFromEnum(InternPool.Index.vector_16_u8_type), + vector_32_u8_type = @intFromEnum(InternPool.Index.vector_32_u8_type), + vector_8_i16_type = @intFromEnum(InternPool.Index.vector_8_i16_type), + vector_16_i16_type = @intFromEnum(InternPool.Index.vector_16_i16_type), + vector_8_u16_type = @intFromEnum(InternPool.Index.vector_8_u16_type), + vector_16_u16_type = @intFromEnum(InternPool.Index.vector_16_u16_type), + vector_4_i32_type = @intFromEnum(InternPool.Index.vector_4_i32_type), + vector_8_i32_type = @intFromEnum(InternPool.Index.vector_8_i32_type), + vector_4_u32_type = @intFromEnum(InternPool.Index.vector_4_u32_type), + vector_8_u32_type = @intFromEnum(InternPool.Index.vector_8_u32_type), + vector_2_i64_type = @intFromEnum(InternPool.Index.vector_2_i64_type), + vector_4_i64_type = @intFromEnum(InternPool.Index.vector_4_i64_type), + vector_2_u64_type = @intFromEnum(InternPool.Index.vector_2_u64_type), + vector_4_u64_type = @intFromEnum(InternPool.Index.vector_4_u64_type), vector_4_f16_type = @intFromEnum(InternPool.Index.vector_4_f16_type), vector_8_f16_type = @intFromEnum(InternPool.Index.vector_8_f16_type), vector_4_f32_type = @intFromEnum(InternPool.Index.vector_4_f32_type), diff --git a/src/InternPool.zig b/src/InternPool.zig index 04783e2a9c..6d0c0a0885 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -4567,6 +4567,22 @@ pub const Index = enum(u32) { slice_const_u8_type, slice_const_u8_sentinel_0_type, + vector_16_i8_type, + vector_32_i8_type, + vector_16_u8_type, + vector_32_u8_type, + vector_8_i16_type, + vector_16_i16_type, + vector_8_u16_type, + vector_16_u16_type, + vector_4_i32_type, + vector_8_i32_type, + vector_4_u32_type, + vector_8_u32_type, + vector_2_i64_type, + vector_4_i64_type, + vector_2_u64_type, + vector_4_u64_type, vector_4_f16_type, vector_8_f16_type, vector_4_f32_type, @@ -5064,6 +5080,38 @@ pub const static_keys = [_]Key{ }, } }, + // @Vector(16, i8) + .{ .vector_type = .{ .len = 16, .child = .i8_type } }, + // @Vector(32, i8) + .{ .vector_type = .{ .len = 32, .child = .i8_type } }, + // @Vector(16, u8) + .{ .vector_type = .{ .len = 16, .child = .u8_type } }, + // @Vector(32, u8) + .{ .vector_type = .{ .len = 32, .child = .u8_type } }, + // @Vector(8, i16) + .{ .vector_type = .{ .len = 8, .child = .i16_type } }, + // @Vector(16, i16) + .{ .vector_type = .{ .len = 16, .child = .i16_type } }, + // @Vector(8, u16) + .{ .vector_type = .{ .len = 8, .child = .u16_type } }, + // @Vector(16, u16) + .{ .vector_type = .{ .len = 16, .child = .u16_type } }, + // @Vector(4, i32) + .{ .vector_type = .{ .len = 4, .child = .i32_type } }, + // @Vector(8, i32) + .{ .vector_type = .{ .len = 8, .child = .i32_type } }, + // @Vector(4, u32) + .{ .vector_type = .{ .len = 4, .child = .u32_type } }, + // @Vector(8, u32) + .{ .vector_type = .{ .len = 8, .child = .u32_type } }, + // @Vector(2, i64) + .{ .vector_type = .{ .len = 2, .child = .i64_type } }, + // @Vector(4, i64) + .{ .vector_type = .{ .len = 4, .child = .i64_type } }, + // @Vector(2, u64) + .{ .vector_type = .{ .len = 2, .child = .u64_type } }, + // @Vector(8, u64) + .{ .vector_type = .{ .len = 4, .child = .u64_type } }, // @Vector(4, f16) .{ .vector_type = .{ .len = 4, .child = .f16_type } }, // @Vector(8, f16) @@ -11703,6 +11751,22 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index { .single_const_pointer_to_comptime_int_type, .slice_const_u8_type, .slice_const_u8_sentinel_0_type, + .vector_16_i8_type, + .vector_32_i8_type, + .vector_16_u8_type, + .vector_32_u8_type, + .vector_8_i16_type, + .vector_16_i16_type, + .vector_8_u16_type, + .vector_16_u16_type, + .vector_4_i32_type, + .vector_8_i32_type, + .vector_4_u32_type, + .vector_8_u32_type, + .vector_2_i64_type, + .vector_4_i64_type, + .vector_2_u64_type, + .vector_4_u64_type, .vector_4_f16_type, .vector_8_f16_type, .vector_4_f32_type, @@ -12026,6 +12090,22 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId { .slice_const_u8_sentinel_0_type, => .pointer, + .vector_16_i8_type, + .vector_32_i8_type, + .vector_16_u8_type, + .vector_32_u8_type, + .vector_8_i16_type, + .vector_16_i16_type, + .vector_8_u16_type, + .vector_16_u16_type, + .vector_4_i32_type, + .vector_8_i32_type, + .vector_4_u32_type, + .vector_8_u32_type, + .vector_2_i64_type, + .vector_4_i64_type, + .vector_2_u64_type, + .vector_4_u64_type, .vector_4_f16_type, .vector_8_f16_type, .vector_4_f32_type, diff --git a/src/Sema.zig b/src/Sema.zig index 3aaf8150b5..5a9272a031 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -36611,6 +36611,22 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value { .single_const_pointer_to_comptime_int_type, .slice_const_u8_type, .slice_const_u8_sentinel_0_type, + .vector_16_i8_type, + .vector_32_i8_type, + .vector_16_u8_type, + .vector_32_u8_type, + .vector_8_i16_type, + .vector_16_i16_type, + .vector_8_u16_type, + .vector_16_u16_type, + .vector_4_i32_type, + .vector_8_i32_type, + .vector_4_u32_type, + .vector_8_u32_type, + .vector_2_i64_type, + .vector_4_i64_type, + .vector_2_u64_type, + .vector_4_u64_type, .vector_4_f16_type, .vector_8_f16_type, .vector_4_f32_type, diff --git a/src/Type.zig b/src/Type.zig index 4379c9c63f..b402d272fb 100644 --- a/src/Type.zig +++ b/src/Type.zig @@ -4175,6 +4175,22 @@ pub const single_const_pointer_to_comptime_int: Type = .{ }; pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_sentinel_0_type }; +pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type }; +pub const vector_32_i8: Type = .{ .ip_index = .vector_32_i8_type }; +pub const vector_16_u8: Type = .{ .ip_index = .vector_16_u8_type }; +pub const vector_32_u8: Type = .{ .ip_index = .vector_32_u8_type }; +pub const vector_8_i16: Type = .{ .ip_index = .vector_8_i16_type }; +pub const vector_16_i16: Type = .{ .ip_index = .vector_16_i16_type }; +pub const vector_8_u16: Type = .{ .ip_index = .vector_8_u16_type }; +pub const vector_16_u16: Type = .{ .ip_index = .vector_16_u16_type }; +pub const vector_4_i32: Type = .{ .ip_index = .vector_4_i32_type }; +pub const vector_8_i32: Type = .{ .ip_index = .vector_8_i32_type }; +pub const vector_4_u32: Type = .{ .ip_index = .vector_4_u32_type }; +pub const vector_8_u32: Type = .{ .ip_index = .vector_8_u32_type }; +pub const vector_2_i64: Type = .{ .ip_index = .vector_2_i64_type }; +pub const vector_4_i64: Type = .{ .ip_index = .vector_4_i64_type }; +pub const vector_2_u64: Type = .{ .ip_index = .vector_2_u64_type }; +pub const vector_4_u64: Type = .{ .ip_index = .vector_4_u64_type }; pub const vector_4_f16: Type = .{ .ip_index = .vector_4_f16_type }; pub const vector_8_f16: Type = .{ .ip_index = .vector_8_f16_type }; pub const vector_4_f32: Type = .{ .ip_index = .vector_4_f32_type }; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 85a3c009d5..3a185acfc1 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2393,7 +2393,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { - @setEvalBranchQuota(2_700); + @setEvalBranchQuota(3_400); const pt = cg.pt; const zcu = pt.zcu; const ip = &zcu.intern_pool; @@ -2433,8 +2433,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .add_wrap, .sub, .sub_wrap, - .min, - .max, => |air_tag| try cg.airBinOp(inst, air_tag), .shr, .shr_exact => try cg.airShlShrBinOp(inst), @@ -2794,6 +2792,5168 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, + .max => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else fallback: { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + if (cg.typeOf(bin_op.lhs).scalarType(zcu).isRuntimeFloat()) break :fallback try cg.airBinOp(inst, air_tag); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._l, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._b, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._l, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._b, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._l, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._b, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._l, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nl, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._b, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nb, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_b, .maxs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._l, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._l, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem } }, + .{ .src = .{ .mem, .to_mut_mmx }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_b, .maxu, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_b, .maxu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .maxu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._b, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._b, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem } }, + .{ .src = .{ .mem, .to_mut_mmx }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .maxs, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .maxs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .maxu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .subus, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_w, .add, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .maxu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .add, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxs, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_d, .maxs, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxs, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxs, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxs, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .maxs, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_d, .maxu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .maxu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .maxu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .maxu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp3x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp4x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp3x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0x, .src1x, .src0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0y, .src1y, .src0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp2y, .tmp1y, ._ }, + .{ ._, .vp_b, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp2x, .tmp1x, ._ }, + .{ ._, .vp_b, .blendv, .tmp1x, .tmp1x, .tmp2x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp1x, .tmp2x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0x, .tmp2x, .dst0x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp3x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0y, .tmp2y, .dst0y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp6y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp6y, .tmp5y, ._ }, + .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp4y, .tmp5y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp6x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp6x, .tmp5x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp4x, .tmp5x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp6x, .tmp5x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp3x, .tmp4x, .tmp6x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._l, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._nl, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src1, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._b, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._nb, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src1, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, + .min => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else fallback: { + const bin_op = air_datas[@intFromEnum(inst)].bin_op; + if (cg.typeOf(bin_op.lhs).scalarType(zcu).isRuntimeFloat()) break :fallback try cg.airBinOp(inst, air_tag); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [1]Temp = undefined; + cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._ge, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._ae, .cmov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword } }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem } }, + .{ .src = .{ .mem, .to_mut_gpr }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .mov, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_signed_int, .any_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nge, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp0p, .tmp1p, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_unsigned_int, .any_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ }, + .{ ._, ._nae, .j, .@"0f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .mem(.src1), ._, ._ }, + .{ .@"0:", ._, .lea, .tmp1p, .mem(.dst0), ._, ._ }, + .{ ._, ._, .mov, .tmp2d, .sa(.src0, .add_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .mins, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_b, .mins, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_b, .cmpgt, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .mins, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .{ .type = .vector_16_i8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movsx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem } }, + .{ .src = .{ .mem, .to_mut_mmx }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_b, .minu, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .minu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_b, .minu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_b, .minu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_b, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_b, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, .slow_incdec, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .movzx, .tmp2d, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .tmp2b, ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1d, .tmp2d, ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .slow_incdec, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0b, .tmp0, .add_size), .tmp1b, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse, .mmx, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .qword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_mmx, .mem } }, + .{ .src = .{ .mem, .to_mut_mmx }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_mmx, .to_mmx } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .mins, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mins, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .mins, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .mins, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .minu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_w, .minu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_w, .subus, .src0x, .src1x, ._, ._ }, + .{ ._, .p_w, .sub, .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_w, .minu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_w, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_w, .subus, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_w, .sub, .tmp2x, .tmp1x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp2x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0w, .tmp0, .add_size), .tmp1w, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mins, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_d, .mins, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .src0x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .mins, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .mins, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .mins, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .mins, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp1x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp2x, .tmp1x, ._, ._ }, + .{ ._, .p_, .andn, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .@"or", .tmp1x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .minu, .dst0x, .src0x, .src1x, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, .p_d, .minu, .dst0x, .src1x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .dst0x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .dst0x, ._, ._ }, + .{ ._, .p_, .xor, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .dst0x, .tmp2x, ._, ._ }, + .{ ._, .p_, .@"and", .src0x, .dst0x, ._, ._ }, + .{ ._, .p_, .andn, .dst0x, .src1x, ._, ._ }, + .{ ._, .p_, .@"or", .dst0x, .src0x, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_d, .minu, .dst0y, .src0y, .src1y, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .minu, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_d, .minu, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._ }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_1, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_d, .minu, .tmp1x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .xword } } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .lea(.xword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, .p_d, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, + .{ ._, .p_, .@"and", .tmp4x, .tmp5x, ._, ._ }, + .{ ._, .p_, .andn, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_, .@"or", .tmp4x, .tmp5x, ._, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp4x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .cmov, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0d, .tmp0, .add_size), .tmp1d, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0x, .src0x, .src1x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, ._dqa, .mov, .tmp0x, .src0x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp0x, .src1x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp0x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_signed_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, .vp_q, .cmpgt, .dst0y, .src0y, .src1y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3y, .tmp1y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .tmp1y, .tmp1y, .tmp2y, .tmp3y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp1y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp3x, .tmp1x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .tmp1x, .tmp1x, .tmp2x, .tmp3x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_i64, .kind = .{ .reg = .xmm0 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp1x, .tmp2x, .tmp3x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .i64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .dst0x, .tmp2x, .src0x, ._ }, + .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .src1x, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0x, .dst0x, .tmp2x, ._ }, + .{ ._, .vp_b, .blendv, .dst0x, .src0x, .src1x, .dst0x }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .mem } }, + .{ .src = .{ .mem, .to_mut_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .ref = .src0 }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .tmp2x, ._, ._ }, + .{ ._, .p_, .xor, .tmp2x, .src0x, ._, ._ }, + .{ ._, .p_, .xor, .tmp3x, .src1x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp2x, .tmp3x, ._, ._ }, + .{ ._, .p_b, .blendv, .dst0x, .src1x, .tmp2x, ._ }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_sse, .mem } }, + .{ .src = .{ .mem, .to_sse }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_sse, .to_sse } }, + }, + .extra_temps = .{ + .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.{ .rc = .sse }}, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, .vp_, .xor, .dst0y, .tmp2y, .src0y, ._ }, + .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .src1y, ._ }, + .{ ._, .vp_q, .cmpgt, .dst0y, .dst0y, .tmp2y, ._ }, + .{ ._, .vp_b, .blendv, .dst0y, .src0y, .src1y, .dst0y }, + } }, + }, .{ + .required_features = .{ .avx2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .yword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_4_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3y, .memia(.src0y, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4y, .memia(.src1y, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5y, .tmp3y, .tmp2y, ._ }, + .{ ._, .vp_, .xor, .tmp6y, .tmp4y, .tmp2y, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5y, .tmp5y, .tmp6y, ._ }, + .{ ._, .vp_b, .blendv, .tmp3y, .tmp3y, .tmp4y, .tmp5y }, + .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_size), .tmp3y, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .avx, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, .v_, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", .v_dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, .v_dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .vp_, .xor, .tmp5x, .tmp3x, .tmp2x, ._ }, + .{ ._, .vp_, .xor, .tmp6x, .tmp4x, .tmp2x, ._ }, + .{ ._, .vp_q, .cmpgt, .tmp5x, .tmp5x, .tmp6x, ._ }, + .{ ._, .vp_b, .blendv, .tmp3x, .tmp3x, .tmp4x, .tmp5x }, + .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .sse4_2, null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .kind = .{ .smin_mem = .{ .ref = .src0, .vectorize_to = .none } } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .{ .type = .vector_2_u64, .kind = .{ .reg = .xmm0 } }, + .{ .type = .vector_2_u64, .kind = .{ .rc = .sse } }, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ }, + .{ ._, ._, .movddup, .tmp2x, .lea(.qword, .tmp0), ._, ._ }, + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._dqa, .mov, .tmp5x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp6x, .tmp2x, ._, ._ }, + .{ ._, ._dqa, .mov, .tmp3x, .memia(.src0x, .tmp0, .add_size), ._, ._ }, + .{ ._, ._dqa, .mov, .tmp4x, .memia(.src1x, .tmp0, .add_size), ._, ._ }, + .{ ._, .p_, .xor, .tmp5x, .tmp3x, ._, ._ }, + .{ ._, .p_, .xor, .tmp6x, .tmp4x, ._, ._ }, + .{ ._, .p_q, .cmpgt, .tmp5x, .tmp6x, ._, ._ }, + .{ ._, .p_b, .blendv, .tmp3x, .tmp4x, .tmp5x, ._ }, + .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp3x, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .cmp, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ .@"1:", ._, .mov, .memia(.dst0q, .tmp0, .add_size), .tmp1q, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._ge, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_signed_int, .any_scalar_signed_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sia(-1, .none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._nge, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memiad(.src1, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memiad(.dst0, .tmp0, .add_size_sub_elem_size, 8), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .cmov, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._ae, .cmov, .tmp1p, .tmp2p, ._, ._ }, + .{ ._, ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .any_scalar_unsigned_int, .any_scalar_unsigned_int }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .isize, .kind = .{ .reg = .rsi } }, + .{ .type = .u64, .kind = .{ .reg = .rdi } }, + .{ .type = .u64, .kind = .{ .reg = .rcx } }, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{.mem}, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"1:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .sbb, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .lea, .tmp0p, .lead(.none, .tmp0, 8), ._, ._ }, + .{ ._, ._c, .de, .tmp1d, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._nae, .j, .@"1f", ._, ._, ._ }, + .{ ._, ._, .lea, .tmp1p, .memia(.src1, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ .@"1:", ._, .lea, .tmp2p, .memia(.dst0, .tmp0, .add_size_sub_elem_size), ._, ._ }, + .{ ._, ._, .mov, .tmp3d, .sa(.none, .add_src0_elem_size_div_8), ._, ._ }, + .{ ._, .@"rep _sq", .mov, ._, ._, ._, ._ }, + .{ ._, ._, .@"test", .tmp0p, .tmp0p, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ + @tagName(air_tag), + cg.typeOf(bin_op.lhs).fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, .alloc => if (use_old) try cg.airAlloc(inst) else { const ty = air_datas[@intFromEnum(inst)].ty; const slot = try cg.tempInit(ty, .{ .lea_frame = .{ @@ -15760,13 +20920,13 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { const src_mcv = try self.resolveInst(ty_op.operand); if (dst_ty.isVector(zcu)) { const max_abi_size = @max(dst_abi_size, src_abi_size); - if (max_abi_size > self.vectorSize(.int)) break :result null; const has_avx = self.hasFeature(.avx); const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu); switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) { .lt => { + if (max_abi_size > self.vectorSize(.int)) break :result null; const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) { else => break :result null, 1 => switch (src_elem_abi_size) { @@ -15823,6 +20983,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { break :result dst_mcv; }, .gt => if (self.hasFeature(.sse4_1)) { + if (max_abi_size > self.vectorSize(.int)) break :result null; const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) { else => break :result null, 2 => if (has_avx) .vp_w else .p_w, @@ -32296,9 +37457,8 @@ fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) ! return mcv; } -fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue { - const pt = self.pt; - return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) { +fn genResult(self: *CodeGen, res: codegen.GenResult) InnerError!MCValue { + return switch (res) { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, @@ -32315,6 +37475,14 @@ fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue { }; } +fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue { + return self.genResult(try codegen.genTypedValue(self.bin_file, self.pt, self.src_loc, val, self.target.*)); +} + +fn lowerUav(self: *CodeGen, val: Value) InnerError!MCValue { + return self.genResult(try self.bin_file.lowerUav(self.pt, val.toIntern(), .none, self.src_loc)); +} + const CallMCValues = struct { args: []MCValue, return_value: InstTracking, @@ -34106,6 +39274,10 @@ fn tempFromValue(cg: *CodeGen, value: Value) !Temp { return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.genTypedValue(value)); } +fn tempMemFromValue(cg: *CodeGen, value: Value) !Temp { + return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerUav(value)); +} + fn tempFromOperand( cg: *CodeGen, inst: Air.Inst.Index, @@ -34364,7 +39536,10 @@ const Select = struct { any_bool_vec, any_int, any_signed_int, + any_unsigned_int, any_scalar_int, + any_scalar_signed_int, + any_scalar_unsigned_int, any_float, po2_any, bool_vec: Memory.Size, @@ -34376,10 +39551,14 @@ const Select = struct { multiple_size: Memory.Size, int: Memory.Size, scalar_int_is: Memory.Size, + scalar_signed_int_is: Memory.Size, + scalar_unsigned_int_is: Memory.Size, scalar_int: OfIsSizes, - scalar_signed_int: Memory.Size, - scalar_unsigned_int: Memory.Size, + scalar_signed_int: OfIsSizes, + scalar_unsigned_int: OfIsSizes, multiple_scalar_int: OfIsSizes, + multiple_scalar_signed_int: OfIsSizes, + multiple_scalar_unsigned_int: OfIsSizes, scalar_remainder_int: OfIsSizes, float: Memory.Size, scalar_any_float: Memory.Size, @@ -34413,7 +39592,10 @@ const Select = struct { .any_bool_vec => ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type, .any_int => cg.intInfo(ty) != null, .any_signed_int => if (cg.intInfo(ty)) |int_info| int_info.signedness == .signed else false, + .any_unsigned_int => if (cg.intInfo(ty)) |int_info| int_info.signedness == .unsigned else false, .any_scalar_int => cg.intInfo(ty.scalarType(zcu)) != null, + .any_scalar_signed_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed else false, + .any_scalar_unsigned_int => if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned else false, .any_float => ty.isRuntimeFloat(), .po2_any => std.math.isPowerOfTwo(ty.abiSize(zcu)), .bool_vec => |size| ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and @@ -34436,18 +39618,30 @@ const Select = struct { size.bitSize(cg.target) >= int_info.bits else false, - .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and - if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, - .scalar_signed_int => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { + .scalar_signed_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { .signed => size.bitSize(cg.target) >= int_info.bits, .unsigned => false, } else false, - .scalar_unsigned_int => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { + .scalar_unsigned_int_is => |size| if (cg.intInfo(ty.scalarType(zcu))) |int_info| switch (int_info.signedness) { .signed => false, .unsigned => size.bitSize(cg.target) >= int_info.bits, } else false, + .scalar_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .scalar_signed_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .scalar_unsigned_int => |of_is| @divExact(of_is.of.bitSize(cg.target), 8) >= cg.unalignedSize(ty) and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, .multiple_scalar_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= int_info.bits else false, + .multiple_scalar_signed_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .signed and + of_is.is.bitSize(cg.target) >= int_info.bits else false, + .multiple_scalar_unsigned_int => |of_is| ty.abiSize(zcu) % @divExact(of_is.of.bitSize(cg.target), 8) == 0 and + if (cg.intInfo(ty.scalarType(zcu))) |int_info| int_info.signedness == .unsigned and + of_is.is.bitSize(cg.target) >= int_info.bits else false, .scalar_remainder_int => |of_is| if (cg.intInfo(ty.scalarType(zcu))) |int_info| of_is.is.bitSize(cg.target) >= (int_info.bits - 1) % of_is.of.bitSize(cg.target) + 1 else @@ -34548,6 +39742,10 @@ const Select = struct { to_x87, mut_x87, to_mut_x87, + mmx, + to_mmx, + mut_mmx, + to_mut_mmx, mm, to_mm, mut_mm, @@ -34611,6 +39809,17 @@ const Select = struct { else => false, }, .to_x87, .to_mut_x87 => true, + .mmx => switch (temp.tracking(cg).short) { + .register => |reg| reg.class() == .mmx, + .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, + else => false, + }, + .mut_mmx => temp.isMut(cg) and switch (temp.tracking(cg).short) { + .register => |reg| reg.class() == .mmx, + .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, + else => false, + }, + .to_mmx, .to_mut_mmx => true, .mm => temp.typeOf(cg).abiSize(cg.pt.zcu) == 8 and switch (temp.tracking(cg).short) { .register => |reg| reg.class() == .mmx, .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, @@ -34668,8 +39877,8 @@ const Select = struct { .mut_gpr, .to_mut_gpr => try temp.toRegClass(true, .general_purpose, cg), .x87, .to_x87 => try temp.toRegClass(false, .x87, cg), .mut_x87, .to_mut_x87 => try temp.toRegClass(true, .x87, cg), - .mm, .to_mm => try temp.toRegClass(false, .mmx, cg), - .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg), + .mmx, .to_mmx, .mm, .to_mm => try temp.toRegClass(false, .mmx, cg), + .mut_mmx, .to_mut_mmx, .mut_mm, .to_mut_mm => try temp.toRegClass(true, .mmx, cg), .sse, .to_sse, .xmm, .to_xmm, .ymm, .to_ymm => try temp.toRegClass(false, .sse, cg), .mut_sse, .to_mut_sse, .mut_xmm, .to_mut_xmm, .mut_ymm, .to_mut_ymm => try temp.toRegClass(true, .sse, cg), }; @@ -34701,7 +39910,7 @@ const Select = struct { umax_mem: ConstInfo, symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 }, - const ConstInfo = struct { ref: Select.Operand.Ref, vectorize_to: Memory.Size = .none }; + const ConstInfo = struct { ref: Select.Operand.Ref, vectorize_to: ?Memory.Size = null }; fn finish(kind: Kind, temp: Temp, s: *const Select) void { switch (kind) { @@ -34758,10 +39967,10 @@ const Select = struct { else => .{ null, ty }, .vector_type => |vector_type| .{ vector_type.len, .fromInterned(vector_type.child) }, }; - const res_vector_len: ?u32 = if (const_info.vectorize_to != .none) - @intCast(@divExact(@divExact(const_info.vectorize_to.bitSize(cg.target), 8), scalar_ty.abiSize(pt.zcu))) - else - vector_len; + const res_vector_len: ?u32 = if (const_info.vectorize_to) |vectorize_to| switch (vectorize_to) { + .none => null, + else => @intCast(@divExact(@divExact(vectorize_to.bitSize(cg.target), 8), scalar_ty.abiSize(pt.zcu))), + } else vector_len; const res_scalar_ty, const res_scalar_val: Value = res_scalar: switch (scalar_ty.toIntern()) { .bool_type => .{ scalar_ty, @@ -34813,7 +40022,7 @@ const Select = struct { })).toIntern(), .storage = .{ .repeated_elem = res_scalar_val.toIntern() }, } })) else res_scalar_val; - return try cg.tempFromValue(res_val); + return try cg.tempMemFromValue(res_val); }, .symbol => |symbol| if (cg.bin_file.cast(.elf)) |elf_file| try cg.tempInit(spec.type, .{ .lea_symbol = .{ .sym_index = try elf_file.getGlobalSymbol(symbol.name, symbol.lib), @@ -34855,13 +40064,13 @@ const Select = struct { mem, }; const Adjust = packed struct(u8) { - factor: i2, - scale: Memory.Scale, - amount: enum(u4) { + sign: enum(u1) { neg, pos }, + lhs: enum(u4) { none, ptr_size, ptr_bit_size, size, + size_sub_elem_size, src0_unaligned_size, bit_size, src0_bit_size, @@ -34874,35 +40083,41 @@ const Select = struct { smax, umax, }, + op: enum(u1) { mul, div }, + rhs: Memory.Scale, - const none: Adjust = .{ .factor = 0, .scale = .@"1", .amount = .none }; - const sub_ptr_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .ptr_size }; - const add_ptr_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .ptr_bit_size }; - const add_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .size }; - const sub_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .size }; - const add_src0_unaligned_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_unaligned_size }; - const sub_src0_unaligned_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_unaligned_size }; - const add_2_bit_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .bit_size }; - const add_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .bit_size }; - const sub_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .bit_size }; - const add_src0_bit_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_bit_size }; - const sub_src0_bit_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_bit_size }; - const add_8_len: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .len }; - const add_4_len: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .len }; - const add_3_len: Adjust = .{ .factor = 1, .scale = .@"3", .amount = .len }; - const add_2_len: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .len }; - const add_len: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .len }; - const sub_len: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .len }; - const add_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size }; - const add_2_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"2", .amount = .src0_elem_size }; - const add_4_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"4", .amount = .src0_elem_size }; - const add_8_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"8", .amount = .src0_elem_size }; - const sub_src0_elem_size: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size }; - const add_src0_elem_size_times_src1: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .src0_elem_size_times_src1 }; - const sub_src0_elem_size_times_src1: Adjust = .{ .factor = -1, .scale = .@"1", .amount = .src0_elem_size_times_src1 }; - const add_log2_src0_elem_size: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .log2_src0_elem_size }; - const add_elem_limbs: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .elem_limbs }; - const add_umax: Adjust = .{ .factor = 1, .scale = .@"1", .amount = .umax }; + const none: Adjust = .{ .sign = .pos, .lhs = .none, .op = .mul, .rhs = .@"1" }; + const sub_ptr_size: Adjust = .{ .sign = .neg, .lhs = .ptr_size, .op = .mul, .rhs = .@"1" }; + const add_ptr_bit_size: Adjust = .{ .sign = .pos, .lhs = .ptr_bit_size, .op = .mul, .rhs = .@"1" }; + const add_size: Adjust = .{ .sign = .pos, .lhs = .size, .op = .mul, .rhs = .@"1" }; + const add_size_div_8: Adjust = .{ .sign = .pos, .lhs = .size, .op = .div, .rhs = .@"8" }; + const sub_size_div_8: Adjust = .{ .sign = .neg, .lhs = .size, .op = .div, .rhs = .@"8" }; + const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" }; + const add_size_sub_elem_size: Adjust = .{ .sign = .pos, .lhs = .size_sub_elem_size, .op = .mul, .rhs = .@"1" }; + const add_src0_unaligned_size: Adjust = .{ .sign = .pos, .lhs = .src0_unaligned_size, .op = .mul, .rhs = .@"1" }; + const sub_src0_unaligned_size: Adjust = .{ .sign = .neg, .lhs = .src0_unaligned_size, .op = .mul, .rhs = .@"1" }; + const add_2_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"2" }; + const add_bit_size: Adjust = .{ .sign = .pos, .lhs = .bit_size, .op = .mul, .rhs = .@"1" }; + const sub_bit_size: Adjust = .{ .sign = .neg, .lhs = .bit_size, .op = .mul, .rhs = .@"1" }; + const add_src0_bit_size: Adjust = .{ .sign = .pos, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" }; + const sub_src0_bit_size: Adjust = .{ .sign = .neg, .lhs = .src0_bit_size, .op = .mul, .rhs = .@"1" }; + const add_8_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"8" }; + const add_4_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"4" }; + const add_3_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"3" }; + const add_2_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"2" }; + const add_len: Adjust = .{ .sign = .pos, .lhs = .len, .op = .mul, .rhs = .@"1" }; + const sub_len: Adjust = .{ .sign = .neg, .lhs = .len, .op = .mul, .rhs = .@"1" }; + const add_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; + const add_2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"2" }; + const add_4_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"4" }; + const add_8_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"8" }; + const add_src0_elem_size_div_8: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size, .op = .div, .rhs = .@"8" }; + const sub_src0_elem_size: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size, .op = .mul, .rhs = .@"1" }; + const add_src0_elem_size_times_src1: Adjust = .{ .sign = .pos, .lhs = .src0_elem_size_times_src1, .op = .mul, .rhs = .@"1" }; + const sub_src0_elem_size_times_src1: Adjust = .{ .sign = .neg, .lhs = .src0_elem_size_times_src1, .op = .mul, .rhs = .@"1" }; + const add_log2_src0_elem_size: Adjust = .{ .sign = .pos, .lhs = .log2_src0_elem_size, .op = .mul, .rhs = .@"1" }; + const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" }; + const add_umax: Adjust = .{ .sign = .pos, .lhs = .umax, .op = .mul, .rhs = .@"1" }; }; const Ref = enum(u4) { tmp0, @@ -35350,11 +40565,15 @@ const Select = struct { const UnsignedImm = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @typeInfo(SignedImm).int.bits }, }); - return @as(i5, op.adjust.factor) * op.adjust.scale.toFactor() * @as(SignedImm, switch (op.adjust.amount) { + const lhs: SignedImm = lhs: switch (op.adjust.lhs) { .none => 0, .ptr_size => @divExact(s.cg.target.ptrBitWidth(), 8), .ptr_bit_size => s.cg.target.ptrBitWidth(), .size => @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu)), + .size_sub_elem_size => { + const ty = op.base.ref.deref(s).typeOf(s.cg); + break :lhs @intCast(ty.abiSize(s.cg.pt.zcu) - ty.elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu)); + }, .src0_unaligned_size => @intCast(s.cg.unalignedSize(Select.Operand.Ref.src0.deref(s).typeOf(s.cg))), .bit_size => @intCast(op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)), .src0_bit_size => @intCast(Select.Operand.Ref.src0.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu)), @@ -35376,7 +40595,20 @@ const Select = struct { .umax => @bitCast(@as(UnsignedImm, std.math.maxInt(UnsignedImm)) >> @truncate( -%op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).bitSize(s.cg.pt.zcu), )), - }) + op.imm; + }; + const rhs = op.adjust.rhs.toLog2(); + const res = res: switch (op.adjust.op) { + .mul => { + const res = @shlWithOverflow(lhs, rhs); + assert(res[1] == 0); + break :res res[0]; + }, + .div => @shrExact(lhs, rhs), + }; + return switch (op.adjust.sign) { + .neg => op.imm - res, + .pos => op.imm + res, + }; } fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand { diff --git a/src/codegen/c/Type.zig b/src/codegen/c/Type.zig index 4aacc7fa5d..53e8a6be82 100644 --- a/src/codegen/c/Type.zig +++ b/src/codegen/c/Type.zig @@ -1437,6 +1437,246 @@ pub const Pool = struct { return pool.fromFields(allocator, .@"struct", &fields, kind); }, + .vector_16_i8_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i8, + .len = 16, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i8.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_32_i8_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i8, + .len = 32, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i8.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_16_u8_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u8, + .len = 16, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_32_u8_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u8, + .len = 32, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_8_i16_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i16, + .len = 8, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i16.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_16_i16_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i16, + .len = 16, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i16.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_8_u16_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u16, + .len = 8, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u16.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_16_u16_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u16, + .len = 16, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u16.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_4_i32_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i32, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i32.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_8_i32_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i32, + .len = 8, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i32.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_4_u32_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u32, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u32.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_8_u32_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u32, + .len = 8, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u32.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_2_i64_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i64, + .len = 2, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i64.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_4_i64_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .i64, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.i64.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_2_u64_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u64, + .len = 2, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u64.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, + .vector_4_u64_type => { + const vector_ctype = try pool.getVector(allocator, .{ + .elem_ctype = .u64, + .len = 4, + }); + if (!kind.isParameter()) return vector_ctype; + var fields = [_]Info.Field{ + .{ + .name = .{ .index = .array }, + .ctype = vector_ctype, + .alignas = AlignAs.fromAbiAlignment(Type.u64.abiAlignment(zcu)), + }, + }; + return pool.fromFields(allocator, .@"struct", &fields, kind); + }, .vector_4_f16_type => { const vector_ctype = try pool.getVector(allocator, .{ .elem_ctype = .f16, diff --git a/test/behavior/maximum_minimum.zig b/test/behavior/maximum_minimum.zig index 53b3d92406..f4601bd580 100644 --- a/test/behavior/maximum_minimum.zig +++ b/test/behavior/maximum_minimum.zig @@ -33,8 +33,7 @@ test "@max on vectors" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_x86_64 and - !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; const S = struct { fn doTheTest() !void { @@ -89,8 +88,7 @@ test "@min for vectors" { if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_x86_64 and - !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; const S = struct { fn doTheTest() !void { @@ -304,7 +302,7 @@ test "@min/@max notices bounds from vector types when element of comptime-known if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and - !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest; + !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; var x: @Vector(2, u32) = .{ 1_000_000, 12345 }; _ = &x; diff --git a/test/behavior/x86_64/math.zig b/test/behavior/x86_64/math.zig index 9547887d38..5ae2b9c769 100644 --- a/test/behavior/x86_64/math.zig +++ b/test/behavior/x86_64/math.zig @@ -1,11 +1,11 @@ const builtin = @import("builtin"); const inf = math.inf; const math = std.math; -const max = math.floatMax; -const min = math.floatMin; +const fmax = math.floatMax; +const fmin = math.floatMin; const nan = math.nan; const std = @import("std"); -const trueMin = math.floatTrueMin; +const tmin = math.floatTrueMin; const Gpr = switch (builtin.cpu.arch) { else => unreachable, @@ -545,96 +545,96 @@ fn Unary(comptime op: anytype) type { fn testFloatTypes() !void { try testArgs(f16, -nan(f16)); try testArgs(f16, -inf(f16)); - try testArgs(f16, -max(f16)); + try testArgs(f16, -fmax(f16)); try testArgs(f16, -10.0); try testArgs(f16, -1.0); try testArgs(f16, -0.1); - try testArgs(f16, -min(f16)); - try testArgs(f16, -trueMin(f16)); + try testArgs(f16, -fmin(f16)); + try testArgs(f16, -tmin(f16)); try testArgs(f16, -0.0); try testArgs(f16, 0.0); - try testArgs(f16, trueMin(f16)); - try testArgs(f16, min(f16)); + try testArgs(f16, tmin(f16)); + try testArgs(f16, fmin(f16)); try testArgs(f16, 0.1); try testArgs(f16, 1.0); try testArgs(f16, 10.0); - try testArgs(f16, max(f16)); + try testArgs(f16, fmax(f16)); try testArgs(f16, inf(f16)); try testArgs(f16, nan(f16)); try testArgs(f32, -nan(f32)); try testArgs(f32, -inf(f32)); - try testArgs(f32, -max(f32)); + try testArgs(f32, -fmax(f32)); try testArgs(f32, -10.0); try testArgs(f32, -1.0); try testArgs(f32, -0.1); - try testArgs(f32, -min(f32)); - try testArgs(f32, -trueMin(f32)); + try testArgs(f32, -fmin(f32)); + try testArgs(f32, -tmin(f32)); try testArgs(f32, -0.0); try testArgs(f32, 0.0); - try testArgs(f32, trueMin(f32)); - try testArgs(f32, min(f32)); + try testArgs(f32, tmin(f32)); + try testArgs(f32, fmin(f32)); try testArgs(f32, 0.1); try testArgs(f32, 1.0); try testArgs(f32, 10.0); - try testArgs(f32, max(f32)); + try testArgs(f32, fmax(f32)); try testArgs(f32, inf(f32)); try testArgs(f32, nan(f32)); try testArgs(f64, -nan(f64)); try testArgs(f64, -inf(f64)); - try testArgs(f64, -max(f64)); + try testArgs(f64, -fmax(f64)); try testArgs(f64, -10.0); try testArgs(f64, -1.0); try testArgs(f64, -0.1); - try testArgs(f64, -min(f64)); - try testArgs(f64, -trueMin(f64)); + try testArgs(f64, -fmin(f64)); + try testArgs(f64, -tmin(f64)); try testArgs(f64, -0.0); try testArgs(f64, 0.0); - try testArgs(f64, trueMin(f64)); - try testArgs(f64, min(f64)); + try testArgs(f64, tmin(f64)); + try testArgs(f64, fmin(f64)); try testArgs(f64, 0.1); try testArgs(f64, 1.0); try testArgs(f64, 10.0); - try testArgs(f64, max(f64)); + try testArgs(f64, fmax(f64)); try testArgs(f64, inf(f64)); try testArgs(f64, nan(f64)); try testArgs(f80, -nan(f80)); try testArgs(f80, -inf(f80)); - try testArgs(f80, -max(f80)); + try testArgs(f80, -fmax(f80)); try testArgs(f80, -10.0); try testArgs(f80, -1.0); try testArgs(f80, -0.1); - try testArgs(f80, -min(f80)); - try testArgs(f80, -trueMin(f80)); + try testArgs(f80, -fmin(f80)); + try testArgs(f80, -tmin(f80)); try testArgs(f80, -0.0); try testArgs(f80, 0.0); - try testArgs(f80, trueMin(f80)); - try testArgs(f80, min(f80)); + try testArgs(f80, tmin(f80)); + try testArgs(f80, fmin(f80)); try testArgs(f80, 0.1); try testArgs(f80, 1.0); try testArgs(f80, 10.0); - try testArgs(f80, max(f80)); + try testArgs(f80, fmax(f80)); try testArgs(f80, inf(f80)); try testArgs(f80, nan(f80)); try testArgs(f128, -nan(f128)); try testArgs(f128, -inf(f128)); - try testArgs(f128, -max(f128)); + try testArgs(f128, -fmax(f128)); try testArgs(f128, -10.0); try testArgs(f128, -1.0); try testArgs(f128, -0.1); - try testArgs(f128, -min(f128)); - try testArgs(f128, -trueMin(f128)); + try testArgs(f128, -fmin(f128)); + try testArgs(f128, -tmin(f128)); try testArgs(f128, -0.0); try testArgs(f128, 0.0); - try testArgs(f128, trueMin(f128)); - try testArgs(f128, min(f128)); + try testArgs(f128, tmin(f128)); + try testArgs(f128, fmin(f128)); try testArgs(f128, 0.1); try testArgs(f128, 1.0); try testArgs(f128, 10.0); - try testArgs(f128, max(f128)); + try testArgs(f128, fmax(f128)); try testArgs(f128, inf(f128)); try testArgs(f128, nan(f128)); } @@ -1391,26 +1391,51 @@ fn Binary(comptime op: anytype) type { ); } fn testIntTypes() !void { + try testArgs(i8, 0x48, 0x6c); try testArgs(u8, 0xbb, 0x43); + try testArgs(i16, -0x0fdf, 0x302e); try testArgs(u16, 0xb8bf, 0x626d); + try testArgs(i32, -0x6280178f, 0x6802c034); try testArgs(u32, 0x80d7a2c6, 0xbff6a402); + try testArgs(i64, 0x0365a53b8ee0c987, -0x1bb6d3013500a7d2); try testArgs(u64, 0x71138bc6b4a38898, 0x1bc4043de9438c7b); + try testArgs(i128, 0x76d428c46cdeaa2ac43de8abffb22f6d, 0x427f7545abe434a12544fdbe2a012889); try testArgs(u128, 0xe05fc132ef2cd8affee00a907f0a851f, 0x29f912a72cfc6a7c6973426a9636da9a); + try testArgs( + i256, + -0x53d4148cee74ea43477a65b3daa7b8fdadcbf4508e793f4af113b8d8da5a7eb6, + -0x30dcbaf7b9b7a3df033694e6795444d842fb0b8f79bc18b3ea8a6b7ccad3ea91, + ); try testArgs( u256, 0xb7935f5c2f3b1ae7a422c0a7c446884294b7d5370bada307d2fe5a4c4284a999, 0x310e6e196ba4f143b8d285ca6addf7f3bb3344224aff221b27607a31e148be08, ); + try testArgs( + i258, + -0x0eee283365108dbeea0bec82f5147418d8ffe86f9eed00e414b4eccd65c21239a, + -0x122c730073fc29a24cd6e3e6263566879bc5325d8566b8db31fcb4a76f7ab95eb, + ); try testArgs( u258, 0x186d5ddaab8cb8cb04e5b41e36f812e039d008baf49f12894c39e29a07796d800, 0x2072daba6ffad168826163eb136f6d28ca4360c8e7e5e41e29755e19e4753a4f5, ); + try testArgs( + i495, + 0x2fe6bc5448c55ce18252e2c9d44777505dfe63ff249a8027a6626c7d8dd9893fd5731e51474727be556f757facb586a4e04bbc0148c6c7ad692302f46fbd, + -0x016a358821ef8240172f3a08e8830c06e6bcf2225f5f4d41ed42b44d249385f55cc594e1278ecac31c73faed890e5054af1a561483bb1bb6fb1f753514cf, + ); try testArgs( u495, 0x6eaf4e252b3bf74b75bac59e0b43ca5326bad2a25b3fdb74a67ef132ac5e47d72eebc3316fb2351ee66c50dc5afb92a75cea9b0e35160652c7db39eeb158, 0x49fbed744a92b549d8c05bb3512c617d24dd824f3f69bdf3923bc326a75674b85f5b828d2566fab9c86f571d12c2a63c9164feb0d191d27905533d09622a, ); + try testArgs( + i512, + -0x3a6876ca92775286c6e1504a64a9b8d56985bebf4a1b66539d404e0e96f24b226f70c4bcff295fdc2043b82513b2052dc45fd78f7e9e80e5b3e101757289f054, + 0x5080c516a819bd32a0a5f0976441bbfbcf89e77684f1f10eb326aeb28e1f8d593278cff60fc99b8ffc87d8696882c64728dd3c322b7142803f4341f85a03bc10, + ); try testArgs( u512, 0xe5b1fedca3c77db765e517aabd05ffc524a3a8aff1784bbf67c45b894447ede32b65b9940e78173c591e56e078932d465f235aece7ad47b7f229df7ba8f12295, @@ -1418,6 +1443,69 @@ fn Binary(comptime op: anytype) type { ); } fn testIntVectorTypes() !void { + try testArgs(@Vector(1, i8), .{ + -0x54, + }, .{ + 0x0f, + }); + try testArgs(@Vector(2, i8), .{ + -0x4d, 0x55, + }, .{ + 0x7d, -0x5d, + }); + try testArgs(@Vector(4, i8), .{ + 0x73, 0x6f, 0x6e, -0x49, + }, .{ + -0x66, 0x23, 0x21, -0x56, + }); + try testArgs(@Vector(8, i8), .{ + 0x44, -0x37, 0x33, -0x2b, -0x1f, 0x3e, 0x50, -0x4d, + }, .{ + 0x6a, 0x1a, -0x0e, 0x4c, -0x46, 0x03, -0x17, 0x3e, + }); + try testArgs(@Vector(16, i8), .{ + -0x52, 0x1a, -0x4b, 0x4e, -0x75, 0x33, -0x43, 0x30, 0x71, -0x30, -0x73, -0x53, 0x64, 0x1f, -0x27, 0x36, + }, .{ + 0x65, 0x77, -0x62, 0x0f, 0x15, 0x52, 0x5c, 0x12, -0x10, 0x36, 0x6d, 0x42, -0x24, -0x79, -0x32, -0x75, + }); + try testArgs(@Vector(32, i8), .{ + -0x12, -0x1e, 0x18, 0x6e, 0x31, 0x53, -0x6a, -0x34, 0x13, 0x4d, 0x30, -0x7d, -0x31, 0x1e, -0x24, 0x32, + -0x1e, -0x01, 0x55, 0x33, -0x75, -0x44, -0x57, 0x2b, -0x66, 0x19, 0x7f, -0x28, -0x3f, -0x7e, -0x5d, -0x06, + }, .{ + 0x05, -0x23, 0x43, -0x54, -0x41, 0x7f, -0x6a, -0x31, 0x04, 0x15, -0x7a, -0x37, 0x6d, 0x16, 0x00, 0x4a, + 0x15, 0x55, -0x4a, 0x16, -0x73, -0x0c, 0x1c, -0x26, -0x14, 0x00, 0x55, 0x7b, 0x16, -0x2e, -0x5f, -0x67, + }); + try testArgs(@Vector(64, i8), .{ + -0x05, 0x76, 0x4e, -0x5c, 0x7b, -0x1a, -0x38, -0x2e, 0x3d, 0x36, 0x01, 0x30, -0x02, -0x71, -0x24, 0x24, + -0x2e, -0x6e, -0x60, 0x74, -0x80, -0x1c, -0x34, -0x08, -0x33, 0x77, 0x1c, -0x0f, 0x45, -0x51, -0x1d, 0x35, + -0x45, 0x44, 0x27, -0x3c, 0x6b, 0x58, -0x6a, -0x26, 0x06, -0x30, -0x21, -0x0a, 0x60, -0x11, -0x05, 0x75, + 0x38, 0x72, -0x6d, -0x1f, -0x7f, 0x74, -0x6b, -0x14, -0x80, 0x35, -0x0f, -0x1e, 0x6a, 0x17, -0x74, -0x6c, + }, .{ + -0x5d, 0x2d, 0x55, 0x40, -0x7c, 0x67, 0x61, 0x5f, 0x14, 0x5b, -0x0c, -0x4d, -0x5f, 0x25, 0x36, 0x3c, + -0x75, -0x48, -0x2b, 0x76, -0x57, -0x4a, 0x1d, 0x65, -0x32, 0x18, -0x2a, -0x0a, -0x6e, -0x3c, -0x62, 0x4e, + -0x24, -0x3c, 0x7d, -0x79, -0x1a, -0x14, -0x03, -0x56, 0x7a, 0x5f, 0x64, -0x68, 0x5f, -0x10, -0x63, -0x07, + 0x79, -0x44, 0x47, 0x7d, 0x6e, 0x77, 0x03, -0x4e, 0x67, 0x38, 0x46, -0x44, -0x41, 0x66, -0x16, -0x0a, + }); + try testArgs(@Vector(128, i8), .{ + 0x30, 0x70, -0x2a, -0x29, -0x35, -0x69, -0x18, 0x2b, 0x4a, -0x17, -0x5f, -0x36, 0x34, -0x26, 0x03, -0x2d, + -0x75, -0x27, -0x07, -0x49, -0x58, 0x00, -0x45, 0x5d, -0x11, -0x68, 0x34, 0x73, -0x4d, 0x7f, -0x25, -0x6a, + 0x46, -0x1d, -0x68, 0x04, 0x64, -0x0d, 0x30, 0x27, -0x24, 0x67, 0x3c, -0x7c, -0x2e, -0x24, 0x24, 0x3e, + -0x2c, -0x05, 0x4e, -0x17, 0x6d, 0x57, 0x76, 0x35, -0x3d, 0x51, 0x71, -0x4e, 0x50, 0x26, 0x4a, -0x42, + 0x73, -0x36, -0x5d, 0x2a, 0x55, 0x33, -0x2b, -0x76, 0x08, 0x43, 0x77, -0x73, -0x0a, 0x5c, -0x03, -0x50, + -0x0a, -0x1c, -0x20, 0x3c, -0x7e, 0x60, 0x11, -0x77, 0x25, -0x71, 0x31, 0x2d, -0x4b, -0x26, -0x2a, 0x7f, + -0x1f, 0x23, -0x34, -0x1f, 0x35, 0x0d, 0x3e, 0x76, -0x08, 0x2c, 0x12, 0x3e, -0x09, -0x3e, 0x4b, -0x52, + -0x1a, -0x44, -0x53, -0x41, -0x6d, -0x5e, -0x06, -0x04, 0x3f, -0x2e, 0x01, 0x54, 0x19, -0x5a, -0x62, -0x3a, + }, .{ + 0x42, -0x11, -0x08, -0x64, -0x55, 0x31, 0x27, -0x66, 0x38, 0x5a, 0x25, -0x68, 0x0b, -0x41, -0x0d, 0x60, + -0x17, -0x6d, 0x62, -0x65, -0x5e, -0x1c, -0x35, 0x28, 0x1c, -0x74, -0x7f, -0x1c, 0x3a, 0x4e, 0x05, -0x08, + 0x30, -0x77, 0x03, 0x68, -0x2c, 0x5c, 0x74, 0x6a, -0x21, 0x0a, 0x36, -0x55, 0x21, 0x29, -0x05, 0x70, + 0x23, 0x3b, 0x0a, 0x7a, 0x19, 0x14, 0x65, -0x1d, 0x2b, 0x65, 0x33, 0x2a, 0x52, -0x63, 0x57, 0x10, + -0x1b, 0x26, -0x46, -0x7e, -0x25, 0x79, -0x01, -0x0d, -0x49, -0x4d, 0x74, 0x03, 0x77, 0x16, 0x03, -0x3d, + 0x1c, 0x25, 0x5a, -0x2f, -0x16, -0x5f, -0x36, -0x55, -0x44, -0x0c, -0x0f, 0x7b, -0x15, -0x1d, 0x32, 0x31, + 0x6e, -0x44, -0x4a, -0x64, 0x67, 0x04, 0x47, 0x00, 0x3c, -0x0a, -0x79, 0x3d, 0x48, 0x5a, 0x61, -0x2c, + 0x6d, -0x68, -0x71, -0x6b, -0x11, 0x44, -0x75, -0x55, -0x67, -0x52, 0x64, -0x3d, -0x05, -0x76, -0x6d, -0x44, + }); + try testArgs(@Vector(1, u8), .{ 0x1f, }, .{ @@ -1481,6 +1569,67 @@ fn Binary(comptime op: anytype) type { 0x56, 0x4f, 0xf1, 0xaa, 0x0a, 0x0f, 0xdb, 0x1b, 0xc8, 0x45, 0x9b, 0x12, 0xb4, 0x1a, 0xe4, 0xa3, }); + try testArgs(@Vector(1, i16), .{ + -0x7b9c, + }, .{ + 0x600a, + }); + try testArgs(@Vector(2, i16), .{ + 0x43cc, -0x1421, + }, .{ + -0x2b0e, 0x4d99, + }); + try testArgs(@Vector(4, i16), .{ + 0x558f, 0x6d92, 0x488f, 0x0a04, + }, .{ + -0x01a9, + 0x2ee4, + 0x24a9, + -0x5fee, + }); + try testArgs(@Vector(8, i16), .{ + -0x7e5d, -0x02e4, -0x3a72, -0x2e30, 0x7c87, 0x3ea0, 0x4f02, 0x06e4, + }, .{ + -0x417f, 0x5a13, -0x117b, 0x4c28, -0x3769, -0x56a8, 0x1656, -0x4431, + }); + try testArgs(@Vector(16, i16), .{ + 0x04be, 0x774a, 0x7395, -0x6ca2, -0x21a0, 0x35be, 0x186c, 0x5991, + -0x1a82, 0x4527, -0x2278, -0x3554, 0x42c1, 0x7f53, -0x670d, 0x1fad, + }, .{ + 0x7a7d, 0x47dd, 0x1975, 0x4028, 0x26ef, -0x24f5, -0x77c9, -0x19a5, + -0x4b04, -0x6939, -0x1b8d, 0x3718, -0x78e6, 0x0941, -0x1208, -0x392d, + }); + try testArgs(@Vector(32, i16), .{ + 0x4cde, 0x3ab0, 0x354e, 0x0bc0, -0x5333, 0x4857, -0x7ccf, -0x69da, + 0x6ab8, 0x2bf3, 0x1c5a, 0x7b11, -0x5653, 0x7bc5, 0x497e, -0x0b55, + 0x7aa8, -0x5a8c, -0x6d05, 0x6210, 0x1b64, 0x3f6f, 0x1a02, 0x65e4, + -0x6795, 0x5867, -0x6faf, -0x07cb, -0x762c, -0x7500, 0x1f1c, -0x4348, + }, .{ + 0x72f6, -0x5405, -0x3aac, 0x2857, 0x34cd, -0x1dce, -0x56d8, 0x7150, + -0x6549, 0x61bd, -0x3a9f, -0x1e02, -0x5a5a, -0x7910, -0x166d, 0x7c8e, + -0x5292, -0x6c6e, -0x37e3, 0x1514, 0x1787, 0x58cb, -0x4d99, -0x6c15, + 0x592e, -0x045f, 0x7682, -0x1eef, 0x1fb2, -0x7117, -0x2a17, -0x2d8e, + }); + try testArgs(@Vector(64, i16), .{ + 0x29c3, -0x1b1f, -0x17ce, -0x50d0, -0x5de3, 0x5ffd, 0x184a, -0x7769, + 0x445e, 0x0d8a, 0x7844, -0x757d, 0x2b32, 0x5374, -0x6ab2, -0x71c4, + 0x38f9, 0x347f, 0x2d4c, 0x69a4, -0x2f92, -0x4479, 0x427b, -0x0c5f, + 0x15ae, 0x2c86, 0x1864, -0x0095, 0x6803, -0x3484, 0x1001, -0x0560, + -0x0824, 0x7bf6, 0x7a3c, -0x458a, -0x65cc, -0x54b1, -0x75c6, 0x782e, + 0x35a7, -0x3188, -0x58ba, 0x40d0, -0x4a9c, 0x6b79, 0x1ef5, 0x67a2, + -0x3fb8, 0x1885, -0x093d, -0x4802, 0x0379, 0x2f52, 0x7f1f, 0x256c, + 0x1052, 0x1b3b, -0x6146, 0x7e0d, 0x79ca, -0x79ee, 0x3d58, 0x7482, + }, .{ + -0x0017, -0x3fdd, -0x6f93, 0x6178, 0x5c2b, 0x4eb3, 0x685b, 0x12c8, + 0x0290, -0x34f4, -0x6572, 0x3ab6, -0x3ed1, -0x5e5f, 0x3a90, -0x4540, + -0x2098, 0x6bde, 0x1246, 0x2212, -0x4d6a, -0x2a5a, 0x5cc4, -0x240f, + 0x51b2, 0x5ec0, -0x5b5f, -0x1b6e, -0x57a5, -0x06bd, -0x5132, 0x7889, + 0x2817, 0x6ada, -0x6b46, -0x6a37, -0x6475, -0x5ff4, 0x5a27, 0x1dfa, + 0x6bd6, -0x49da, -0x09bf, -0x7c53, 0x2cd3, -0x6be0, -0x2dca, 0x44bd, + -0x1b95, 0x7680, -0x5bb0, 0x7ad7, -0x1988, 0x149f, 0x631e, -0x1d2d, + 0x632b, 0x55c7, -0x3433, 0x0dde, -0x27a7, 0x560e, -0x2063, 0x4570, + }); + try testArgs(@Vector(1, u16), .{ 0x9d6f, }, .{ @@ -1539,6 +1688,59 @@ fn Binary(comptime op: anytype) type { 0x2c02, 0xff5b, 0x19ca, 0xbbf5, 0x870e, 0xc9ca, 0x47bb, 0xcfcc, }); + try testArgs(@Vector(1, i32), .{ + 0x7aef7b1e, + }, .{ + 0x60310858, + }); + try testArgs(@Vector(2, i32), .{ + -0x21910ac9, 0x669f37ef, + }, .{ + 0x1a2a1681, 0x003b1fdf, + }); + try testArgs(@Vector(4, i32), .{ + 0x7906cf0d, 0x4818a45f, -0x0a2833b6, 0x51a018c9, + }, .{ + -0x05a3e6a7, -0x47f4a500, 0x50d1141f, -0x264c85c2, + }); + try testArgs(@Vector(8, i32), .{ + 0x7566235a, -0x7720144f, -0x7d4f5489, 0x3cd736c8, + -0x77388801, 0x4e7f955a, 0x4cdf52bc, 0x50b0b53f, + }, .{ + 0x00ed6fc5, 0x37320361, 0x70c563c2, -0x09acb495, + 0x0688e83f, 0x797295c4, -0x23bfbfdb, 0x38552096, + }); + try testArgs(@Vector(16, i32), .{ + -0x0214589d, 0x74a7537f, 0x7a7dcb26, 0x3e2e4c44, + -0x23bfc358, 0x60e8ef18, 0x5524a7bc, -0x3d88c153, + -0x7dc8ff0f, 0x6e2698f6, 0x05641ab8, -0x45e9e405, + -0x7c1a04d0, -0x4a8d1e91, 0x41d56723, 0x4ba924ab, + }, .{ + -0x528dc756, -0x6bc217f4, 0x40789b06, 0x65f08d3a, + -0x077140ea, -0x43bdaa79, 0x5d98f4e7, -0x2356a1ca, + -0x36ef2b49, -0x7cd09b06, 0x71c8176e, 0x5b005860, + 0x6ce8cfab, -0x49fd7609, 0x6cbb4e33, 0x6c7c121d, + }); + try testArgs(@Vector(32, i32), .{ + 0x7d22905d, -0x354e4bbe, -0x68662618, -0x246e1858, + -0x1c4285a9, -0x0338059c, -0x60f5bbf4, -0x04f06917, + -0x55f837b6, -0x2fba5fe3, 0x092aabf4, -0x5f533b31, + 0x6e81a558, -0x7bcac358, 0x6c4d8d04, 0x3e2f9852, + -0x78589b1a, -0x68a00fd4, -0x77d55e25, 0x7f79b51c, + -0x66b88f45, 0x7f6dc8a5, -0x27299a82, -0x426c8e1c, + 0x0c288f16, 0x158f8c3f, 0x26708be1, -0x0b73626e, + -0x32df1bee, 0x196330f4, -0x68bb9529, -0x26376ab6, + }, .{ + 0x63bd0bd4, 0x4e507611, -0x5e5222b8, -0x35d8e114, + 0x1feab77b, -0x20de7dfd, -0x0ed0b09f, -0x7fc3d585, + -0x2d3018e9, -0x261d431b, 0x54451864, 0x1415288f, + -0x3ab89593, -0x7060e4c1, -0x54fcd501, -0x26324630, + 0x53fc8294, 0x2d4aceef, -0x4ac8efd2, -0x2fec97b7, + -0x4de3a2fc, 0x2269fe52, -0x58c8b473, -0x21026285, + -0x23438776, 0x3d5c8c41, -0x1fc946b2, -0x161c7005, + 0x44913ff1, -0x76e2bfaa, -0x54636350, -0x6ec53870, + }); + try testArgs(@Vector(1, u32), .{ 0x1d0d9cc4, }, .{ @@ -1592,6 +1794,54 @@ fn Binary(comptime op: anytype) type { 0xf080e943, 0xc8718d14, 0x3f920382, 0x18d101b5, }); + try testArgs(@Vector(1, i64), .{ + 0x4a31679b316d8b59, + }, .{ + 0x34a583368386afde, + }); + try testArgs(@Vector(2, i64), .{ + 0x3bae373f9cb990b3, -0x7e8c6c876e8fd34a, + }, .{ + 0x09dbef6f7cb9c726, 0x48dfeca879b0df51, + }); + try testArgs(@Vector(4, i64), .{ + -0x2bd24dd5f5da94bf, -0x144113bae33082c2, + 0x51e8cb7027ba4b12, -0x47b02168e2e22f13, + }, .{ + 0x769f113245641b91, -0x414d0e24ea97bc53, + -0x0d2a570e7ef9e923, -0x070513d46d3b5a4c, + }); + try testArgs(@Vector(8, i64), .{ + 0x10bb6779b6a55ca9, 0x5f6ffd567a187af4, + -0x6ba191b1168486b4, -0x441b92ce455870a1, + 0x2b6fdefbec9386ad, -0x6fdd3938d79217e4, + 0x6aa8fe1fb891501f, 0x20802f5bbdf6dc50, + }, .{ + -0x7500319df437b479, 0x00ceb712d4fa62d4, + 0x67e715b9e99e660d, -0x17ae00e1f0009ec2, + -0x5b700b948503acdf, -0x3ff61fb5cce5a530, + 0x55a3efac2e3694a4, 0x7f951a8d842f1670, + }); + try testArgs(@Vector(16, i64), .{ + 0x37a205109a685810, -0x50ff5d13134ccaa6, + 0x26813391c5505d5d, -0x502cdc01603a2f21, + -0x6b1b44b1c850c7ea, 0x1f6db974ace9dd70, + -0x47d15da8b519e328, 0x3ac0763abbf79d8d, + 0x5f12e0dc1aed4a4f, -0x46a973e16061e928, + -0x3f59a3fa9699b4d5, -0x2f5012d390c78315, + -0x40e510dea2c47e9c, 0x221c51defe0acc9a, + -0x385fd6f1d390b84b, 0x35932fe2783fa6b9, + }, .{ + 0x0ba5202b71ad73dd, 0x65c8d2d5e2a14fe5, + 0x2e4d97cd66c41a3d, 0x14babbb47da51193, + 0x59d1d12b42ade3aa, -0x3c3617e556dfa8fb, + -0x5a36602ba43279c4, -0x61f1ddda13665d9f, + -0x50cd6128589ddd04, 0x135ae0dcc85674ae, + -0x25e80592affc038d, 0x07e184c44fbe9b12, + -0x70ede1b90964bbaa, 0x3ec48b32e8efd98e, + -0x5267d41d85a29f46, 0x53099805f9116b60, + }); + try testArgs(@Vector(1, u64), .{ 0x333f593bf9d08546, }, .{ @@ -1640,6 +1890,49 @@ fn Binary(comptime op: anytype) type { 0xed533d18f8657f3f, 0x1ddd7cd7f6bab957, }); + try testArgs(@Vector(1, i128), .{ + -0x3bb56309fcad13fc1011dc671cf57bdc, + }, .{ + -0x05338bb517db516ee08c45d1408e5836, + }); + try testArgs(@Vector(2, i128), .{ + 0x295f2901e3837e5592b9435f8c4df8a7, + -0x1f246b0ff2d02a6bf30a63392fc63371, + }, .{ + -0x31060c09e29b545670c4cbc721a4e26b, + -0x631eb286321325d51c617aa798195392, + }); + try testArgs(@Vector(4, i128), .{ + 0x47110102c74f620f08e5b7c5dbe193c2, + -0x61d12d2650413ad3ffeeeab3ba57e1f0, + 0x449781e64b29dc8a17a88f4b7a5b0717, + 0x0d2170e9238d12a585dc5377566e1938, + }, .{ + 0x0bf948e19bd01823dcb3887937d97079, + -0x16f933ab12bfba3560d0d39ffe69b64a, + 0x3d0bfce3907a5cd157348f0329e2548e, + -0x3c2d182e2e238a4bebd7defbd7f9699a, + }); + try testArgs(@Vector(8, i128), .{ + -0x775678727c721662f02480619acbfc82, + -0x6f504fcbff673cb91e4706af4373665f, + -0x670f888d4186387c3106d125b856c294, + 0x0641e7efdfdd924d126b446d874154f8, + 0x57d7aef0f82d3351917f43c8f677392b, + -0x4077e745dede8367d145c94f20ab8810, + -0x0344a74fb60e1f1f72ba8ec288b05939, + -0x0be3ce9be461aca1d25ad8e74dcc36e1, + }, .{ + -0x4a873d91e5a2331def0d34c008d33d83, + 0x2744cecfd4c683bdd12f3cfc11d7f520, + -0x0cb8e468fc1de93a7c5ad2a5a61e8f50, + -0x1a3be9e58e918d6586cc4948a54515d3, + -0x512ec6f88c3a34950a8aaee47130120b, + -0x2e772e4a8812e553bcf9b2754a493709, + 0x0c7b137937dc25f9f9cbaf4d7a88ee6b, + -0x2ecdd5eb81eb0e98ed8d0aa9516c1617, + }); + try testArgs(@Vector(1, u128), .{ 0x5f11e16b0ca3392f907a857881455d2e, }, .{ @@ -1683,6 +1976,30 @@ fn Binary(comptime op: anytype) type { 0xa858bce5ad0e48c13588a4e170e8667c, }); + try testArgs(@Vector(1, i256), .{ + 0x1fe30aed39db1accf4d1b43845aec28c1094b500492555fdf59b4f2f85c6a1ce, + }, .{ + 0x6932f4faf261c45ecd701a4fe3015d4255e486b04c4ab448fe162980cead63fb, + }); + try testArgs(@Vector(2, i256), .{ + -0x23daa9bab59dc1e685f4220c189930c3420a55784f0dec1028c2778d907ccfe2, + 0x521c992e4f46d61709d39e076ed94d5d884585f85ccbf71ca4d593da34f61bf5, + }, .{ + 0x2d880cb5aa793218a32411389db31e935932029645573a9625dd174099c9e5b2, + 0x2394a6cde7e8b2dc2995f07f22f815baa6c223d99c0b1ec4b2d8abd0094db853, + }); + try testArgs(@Vector(4, i256), .{ + 0x244e66ed932a4d970fd8735c10bfbd5f59bd4452c20fa0fcf873823b8c9e6321, + -0x31577b747614b1ab83fd0178293cd80b3cb92e739459b2d038688a2471f6d659, + -0x0dbdfc3d8bbd7cab6a33598cef29125aab7571fb0db9a528e42966963d6ce0e7, + -0x72c58cce172d8a34019a44407a4baf1f8f8a4a611711bd5bb4daa2a2739dd67b, + }, .{ + -0x2e88bc68893fc2d61af0e5ccb541f31fa6169504e8cfcbeab0b74a03b9e86c33, + -0x7eba0783f3382b59a17ffbea57ba1dd8fa30e2d4f7eba7ed68d336d3c37b4561, + -0x66d1463efd38e9e994e126d09b5c65c8efc932ffea9ec6cdf6042561ba05f801, + 0x2024bbacefbabbfd5b32a09be631451764a1f889a77918f9094382dc6d02aef2, + }); + try testArgs(@Vector(1, u256), .{ 0x28df37e1f57a56133ba3f5b5b2164ce24eb6c29a8973a597fd91fbee8ab4bafb, }, .{ @@ -1707,6 +2024,19 @@ fn Binary(comptime op: anytype) type { 0xe47a122bd45d5e7d69722d864a6b795ddee965a0993094f8791dd309d692de8b, }); + try testArgs(@Vector(1, i512), .{ + -0x439ba81b44584e0c4d7abc80d18ab9d679a4e921884e877b28d04eb15b2d3e7be8d670b0aba2c4cc25c12655e1899ab514d0a6e50a221bcf076d506e6411d5c2, + }, .{ + 0x18b1d3be5a03310d82859a4ab72f056a33d1a4b554522bcc062fb33eda3b8111045ee79e045dd1a665d250b897f6f2e12003a03313c2547698f8c1eab452eae1, + }); + try testArgs(@Vector(2, i512), .{ + 0x28e2ab84d87d5fb12be65d8650de67b992dd162fe563ca74b62f51f2f32e1084e03e32c8370930816445ac5052b4d345059c8ace582e3ef44377b160e265ec9b, + -0x3a96548c707219326c42063997e71bc7a17b3067d402063843f84c86e747b71e09338079c28943d20601c0cde018bad57f5615fc89784bcb6232e45c54dff1db, + }, .{ + 0x64beecc90609b7156653b75a861e174c58fb42d5c7bf8d793efbb1cbe785c6b8cd52ce5f9aa859f174123c387820d40a2f93122b81396d739eb85c3ea33fcd37, + -0x3632e347bc6d794940424ca0945dafa04328a924ec6b0ccdedcda6d296e09aa2dd5dca83b934cac752993238aa4fe826be8d62991c9347bae6f01bc0b1b4223d, + }); + try testArgs(@Vector(1, u512), .{ 0x651058c1d89a8f34cfc5e66b6d25294eecfcc4a7e1e4a356eb51ee7d7b2db25378e4afee51b7d18d16e520772a60c50a02d7966f40ced1870b32c658e5821397, }, .{ @@ -1720,6 +2050,12 @@ fn Binary(comptime op: anytype) type { 0x8fa3d86099e9e2789d72f8e792290356d659ab20ac0414ff94745984c6ae7d986082197bb849889f912e896670aa2c1a11bd7e66e3f650710b0f0a18a1533f90, }); + try testArgs(@Vector(1, i1024), .{ + -0x4fe568569c0531c9bfbbda1516e93a6c61a3d035c98e13fdc85225165a3bea84d5dc6b610ced008f9321453af42ea50bbf6881d40d2759b73b9b6186c0d6d243f367e292cbbf6b5c5c30d7f4e8de19701c7b0fc9e67cdf31228daa1675a4887f6c4f1588b48855d6f4730a21f27dec8a756c568727709b65cd531020d53ff394, + }, .{ + -0x7cab2a053dfbf944cd342460350c989fd1b4469a6c7b54ddcacd54e605d29c03651b5c463495610d82269c9ac5b51bfd07816a0f7b1ab50cb598989ed64607b3faff79a190702eb285b0fedc050ec1a71537abc47ec590eb671d4f76b19567049ba4789d1a4348385607a0320fbff9b78260536a9b6030bddb0b09da689d1687, + }); + try testArgs(@Vector(1, u1024), .{ 0x0ca1a0dfaf8bb1da714b457d23c71aef948e66c7cd45c0aa941498a796fb18502ec32f34e885d0a107d44ae81595f8b52c2f0fb38e584b7139903a0e8a823ae20d01ca0662722dd474e7efc40f32d74cc065d97d8a09d0447f1ab6107fa0a57f3f8c866ae872506627ce82f18add79cee8dc69837f4ead3ca770c4d622d7e544, }, .{ @@ -1778,3 +2114,19 @@ test bitXor { try Binary(bitXor).testIntTypes(); try Binary(bitXor).testIntVectorTypes(); } + +inline fn min(comptime Type: type, lhs: Type, rhs: Type) Type { + return @min(lhs, rhs); +} +test min { + try Binary(min).testIntTypes(); + try Binary(min).testIntVectorTypes(); +} + +inline fn max(comptime Type: type, lhs: Type, rhs: Type) Type { + return @max(lhs, rhs); +} +test max { + try Binary(max).testIntTypes(); + try Binary(max).testIntVectorTypes(); +} diff --git a/test/cases/compile_errors/anytype_param_requires_comptime.zig b/test/cases/compile_errors/anytype_param_requires_comptime.zig index e10511a2f9..35d68d19b8 100644 --- a/test/cases/compile_errors/anytype_param_requires_comptime.zig +++ b/test/cases/compile_errors/anytype_param_requires_comptime.zig @@ -15,6 +15,6 @@ pub export fn entry() void { // error // // :7:25: error: unable to resolve comptime value -// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_447.C' must be comptime-known +// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_463.C' must be comptime-known // :4:16: note: struct requires comptime because of this field // :4:16: note: types are not available at runtime diff --git a/test/cases/compile_errors/bogus_method_call_on_slice.zig b/test/cases/compile_errors/bogus_method_call_on_slice.zig index 51524416c0..1a8c8830df 100644 --- a/test/cases/compile_errors/bogus_method_call_on_slice.zig +++ b/test/cases/compile_errors/bogus_method_call_on_slice.zig @@ -16,5 +16,5 @@ pub export fn entry2() void { // // :3:6: error: no field or member function named 'copy' in '[]const u8' // :9:8: error: no field or member function named 'bar' in '@TypeOf(.{})' -// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_451' +// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_467' // :12:6: note: struct declared here diff --git a/test/cases/compile_errors/coerce_anon_struct.zig b/test/cases/compile_errors/coerce_anon_struct.zig index c043e190e7..21f199f622 100644 --- a/test/cases/compile_errors/coerce_anon_struct.zig +++ b/test/cases/compile_errors/coerce_anon_struct.zig @@ -6,6 +6,6 @@ export fn foo() void { // error // -// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_440' +// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_456' // :3:16: note: struct declared here // :1:11: note: struct declared here