diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index e20cf900af..e2e2ce9ead 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -3011,41 +3011,16 @@ fn optionalPayload(self: *Self, inst: Air.Inst.Index, mcv: MCValue, optional_ty: return MCValue{ .register = reg }; } - const offset = @intCast(u32, optional_ty.abiSize(self.target.*) - payload_ty.abiSize(self.target.*)); switch (mcv) { - .register => |source_reg| { + .register => { // TODO should we reuse the operand here? const raw_reg = try self.register_manager.allocReg(inst, gp); const dest_reg = raw_reg.toX(); - const shift = @intCast(u6, offset * 8); - if (shift == 0) { - try self.genSetReg(payload_ty, dest_reg, mcv); - } else { - _ = try self.addInst(.{ - .tag = if (payload_ty.isSignedInt()) - Mir.Inst.Tag.asr_immediate - else - Mir.Inst.Tag.lsr_immediate, - .data = .{ .rr_shift = .{ - .rd = dest_reg, - .rn = source_reg.toX(), - .shift = shift, - } }, - }); - } - + try self.genSetReg(payload_ty, dest_reg, mcv); return MCValue{ .register = self.registerAlias(dest_reg, payload_ty) }; }, - .stack_argument_offset => |off| { - return MCValue{ .stack_argument_offset = off + offset }; - }, - .stack_offset => |off| { - return MCValue{ .stack_offset = off - offset }; - }, - .memory => |addr| { - return MCValue{ .memory = addr + offset }; - }, + .stack_argument_offset, .stack_offset, .memory => return mcv, else => unreachable, // invalid MCValue for an error union } } @@ -3289,12 +3264,11 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void { const optional_abi_size = @intCast(u32, optional_ty.abiSize(self.target.*)); const optional_abi_align = optional_ty.abiAlignment(self.target.*); - const payload_abi_size = @intCast(u32, payload_ty.abiSize(self.target.*)); - const offset = optional_abi_size - payload_abi_size; + const offset = @intCast(u32, payload_ty.abiSize(self.target.*)); const stack_offset = try self.allocMem(optional_abi_size, optional_abi_align, inst); - try self.genSetStack(Type.bool, stack_offset, .{ .immediate = 1 }); - try self.genSetStack(payload_ty, stack_offset - @intCast(u32, offset), operand); + try self.genSetStack(payload_ty, stack_offset, operand); + try self.genSetStack(Type.bool, stack_offset - offset, .{ .immediate = 1 }); break :result MCValue{ .stack_offset = stack_offset }; }; @@ -4834,13 +4808,49 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { } fn isNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue { - const sentinel_ty: Type = if (!operand_ty.isPtrLikeOptional()) blk: { + const sentinel: struct { ty: Type, bind: ReadArg.Bind } = if (!operand_ty.isPtrLikeOptional()) blk: { var buf: Type.Payload.ElemType = undefined; const payload_ty = operand_ty.optionalChild(&buf); - break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else operand_ty; - } else operand_ty; + if (!payload_ty.hasRuntimeBitsIgnoreComptime()) + break :blk .{ .ty = operand_ty, .bind = operand_bind }; + + const offset = @intCast(u32, payload_ty.abiSize(self.target.*)); + const operand_mcv = try operand_bind.resolveToMcv(self); + const new_mcv: MCValue = switch (operand_mcv) { + .register => |source_reg| new: { + // TODO should we reuse the operand here? + const raw_reg = try self.register_manager.allocReg(null, gp); + const dest_reg = raw_reg.toX(); + + const shift = @intCast(u6, offset * 8); + if (shift == 0) { + try self.genSetReg(payload_ty, dest_reg, operand_mcv); + } else { + _ = try self.addInst(.{ + .tag = if (payload_ty.isSignedInt()) + Mir.Inst.Tag.asr_immediate + else + Mir.Inst.Tag.lsr_immediate, + .data = .{ .rr_shift = .{ + .rd = dest_reg, + .rn = source_reg.toX(), + .shift = shift, + } }, + }); + } + + break :new .{ .register = self.registerAlias(dest_reg, payload_ty) }; + }, + .stack_argument_offset => |off| .{ .stack_argument_offset = off + offset }, + .stack_offset => |off| .{ .stack_offset = off - offset }, + .memory => |addr| .{ .memory = addr + offset }, + else => unreachable, // invalid MCValue for an optional + }; + + break :blk .{ .ty = Type.bool, .bind = .{ .mcv = new_mcv } }; + } else .{ .ty = operand_ty, .bind = operand_bind }; const imm_bind: ReadArg.Bind = .{ .mcv = .{ .immediate = 0 } }; - return self.cmp(operand_bind, imm_bind, sentinel_ty, .eq); + return self.cmp(sentinel.bind, imm_bind, sentinel.ty, .eq); } fn isNonNull(self: *Self, operand_bind: ReadArg.Bind, operand_ty: Type) !MCValue { diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index c05f07a602..9af66eb40c 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -2715,20 +2715,7 @@ fn lowerParentPtr(func: *CodeGen, ptr_val: Value, ptr_child_ty: Type) InnerError }, .opt_payload_ptr => { const payload_ptr = ptr_val.castTag(.opt_payload_ptr).?.data; - const parent_ptr = try func.lowerParentPtr(payload_ptr.container_ptr, payload_ptr.container_ty); - var buf: Type.Payload.ElemType = undefined; - const payload_ty = payload_ptr.container_ty.optionalChild(&buf); - if (!payload_ty.hasRuntimeBitsIgnoreComptime() or payload_ty.optionalReprIsPayload()) { - return parent_ptr; - } - - const abi_size = payload_ptr.container_ty.abiSize(func.target); - const offset = abi_size - payload_ty.abiSize(func.target); - - return WValue{ .memory_offset = .{ - .pointer = parent_ptr.memory, - .offset = @intCast(u32, offset), - } }; + return func.lowerParentPtr(payload_ptr.container_ptr, payload_ptr.container_ty); }, else => |tag| return func.fail("TODO: Implement lowerParentPtr for tag: {}", .{tag}), } @@ -2889,7 +2876,7 @@ fn lowerConstant(func: *CodeGen, arg_val: Value, ty: Type) InnerError!WValue { } } else { const is_pl = val.tag() == .opt_payload; - return WValue{ .imm32 = if (is_pl) @as(u32, 1) else 0 }; + return WValue{ .imm32 = @boolToInt(is_pl) }; }, .Struct => { const struct_obj = ty.castTag(.@"struct").?.data; @@ -3882,7 +3869,11 @@ fn isNull(func: *CodeGen, operand: WValue, optional_ty: Type, opcode: wasm.Opcod // When payload is zero-bits, we can treat operand as a value, rather than // a pointer to the stack value if (payload_ty.hasRuntimeBitsIgnoreComptime()) { - try func.addMemArg(.i32_load8_u, .{ .offset = operand.offset(), .alignment = 1 }); + const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse { + const module = func.bin_file.base.options.module.?; + return func.fail("Optional type {} too big to fit into stack frame", .{optional_ty.fmt(module)}); + }; + try func.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 }); } } else if (payload_ty.isSlice()) { switch (func.arch()) { @@ -3911,13 +3902,11 @@ fn airOptionalPayload(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const operand = try func.resolveInst(ty_op.operand); if (opt_ty.optionalReprIsPayload()) break :result func.reuseOperand(ty_op.operand, operand); - const offset = opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target); - if (isByRef(payload_ty, func.target)) { - break :result try func.buildPointerOffset(operand, offset, .new); + break :result try func.buildPointerOffset(operand, 0, .new); } - const payload = try func.load(operand, payload_ty, @intCast(u32, offset)); + const payload = try func.load(operand, payload_ty, 0); break :result try payload.toLocal(func, payload_ty); }; func.finishAir(inst, result, &.{ty_op.operand}); @@ -3936,8 +3925,7 @@ fn airOptionalPayloadPtr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { break :result func.reuseOperand(ty_op.operand, operand); } - const offset = opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target); - break :result try func.buildPointerOffset(operand, offset, .new); + break :result try func.buildPointerOffset(operand, 0, .new); }; func.finishAir(inst, result, &.{ty_op.operand}); } @@ -3956,16 +3944,16 @@ fn airOptionalPayloadPtrSet(func: *CodeGen, inst: Air.Inst.Index) InnerError!voi return func.finishAir(inst, operand, &.{ty_op.operand}); } - const offset = std.math.cast(u32, opt_ty.abiSize(func.target) - payload_ty.abiSize(func.target)) orelse { + const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse { const module = func.bin_file.base.options.module.?; return func.fail("Optional type {} too big to fit into stack frame", .{opt_ty.fmt(module)}); }; try func.emitWValue(operand); try func.addImm32(1); - try func.addMemArg(.i32_store8, .{ .offset = operand.offset(), .alignment = 1 }); + try func.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 }); - const result = try func.buildPointerOffset(operand, offset, .new); + const result = try func.buildPointerOffset(operand, 0, .new); return func.finishAir(inst, result, &.{ty_op.operand}); } @@ -3988,7 +3976,7 @@ fn airWrapOptional(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { if (op_ty.optionalReprIsPayload()) { break :result func.reuseOperand(ty_op.operand, operand); } - const offset = std.math.cast(u32, op_ty.abiSize(func.target) - payload_ty.abiSize(func.target)) orelse { + const offset = std.math.cast(u32, payload_ty.abiSize(func.target)) orelse { const module = func.bin_file.base.options.module.?; return func.fail("Optional type {} too big to fit into stack frame", .{op_ty.fmt(module)}); }; @@ -3997,9 +3985,9 @@ fn airWrapOptional(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result_ptr = try func.allocStack(op_ty); try func.emitWValue(result_ptr); try func.addImm32(1); - try func.addMemArg(.i32_store8, .{ .offset = result_ptr.offset(), .alignment = 1 }); + try func.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 }); - const payload_ptr = try func.buildPointerOffset(result_ptr, offset, .new); + const payload_ptr = try func.buildPointerOffset(result_ptr, 0, .new); try func.store(payload_ptr, operand, payload_ty, 0); break :result result_ptr; }; @@ -4719,7 +4707,6 @@ fn cmpOptionals(func: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: assert(op == .eq or op == .neq); var buf: Type.Payload.ElemType = undefined; const payload_ty = operand_ty.optionalChild(&buf); - const offset = @intCast(u32, operand_ty.abiSize(func.target) - payload_ty.abiSize(func.target)); // We store the final result in here that will be validated // if the optional is truly equal. @@ -4732,8 +4719,8 @@ fn cmpOptionals(func: *CodeGen, lhs: WValue, rhs: WValue, operand_ty: Type, op: try func.addTag(.i32_ne); // inverse so we can exit early try func.addLabel(.br_if, 0); - _ = try func.load(lhs, payload_ty, offset); - _ = try func.load(rhs, payload_ty, offset); + _ = try func.load(lhs, payload_ty, 0); + _ = try func.load(rhs, payload_ty, 0); const opcode = buildOpcode(.{ .op = .ne, .valtype1 = typeToValtype(payload_ty, func.target) }); try func.addTag(Mir.Inst.Tag.fromOpcode(opcode)); try func.addLabel(.br_if, 0); diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 5dfce901f7..f30be0e378 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -402,19 +402,38 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 { fn asmSetccRegister(self: *Self, reg: Register, cc: bits.Condition) !void { _ = try self.addInst(.{ .tag = .setcc, - .ops = .r_c, - .data = .{ .r_c = .{ + .ops = .r_cc, + .data = .{ .r_cc = .{ .r1 = reg, .cc = cc, } }, }); } +fn asmSetccMemory(self: *Self, m: Memory, cc: bits.Condition) !void { + _ = try self.addInst(.{ + .tag = .setcc, + .ops = switch (m) { + .sib => .m_sib_cc, + .rip => .m_rip_cc, + else => unreachable, + }, + .data = .{ .x_cc = .{ + .payload = switch (m) { + .sib => try self.addExtra(Mir.MemorySib.encode(m)), + .rip => try self.addExtra(Mir.MemoryRip.encode(m)), + else => unreachable, + }, + .cc = cc, + } }, + }); +} + fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bits.Condition) !void { _ = try self.addInst(.{ .tag = .cmovcc, - .ops = .rr_c, - .data = .{ .rr_c = .{ + .ops = .rr_cc, + .data = .{ .rr_cc = .{ .r1 = reg1, .r2 = reg2, .cc = cc, @@ -422,6 +441,26 @@ fn asmCmovccRegisterRegister(self: *Self, reg1: Register, reg2: Register, cc: bi }); } +fn asmCmovccRegisterMemory(self: *Self, reg: Register, m: Memory, cc: bits.Condition) !void { + _ = try self.addInst(.{ + .tag = .cmovcc, + .ops = switch (m) { + .sib => .rm_sib_cc, + .rip => .rm_rip_cc, + else => unreachable, + }, + .data = .{ .rx_cc = .{ + .r1 = reg, + .cc = cc, + .payload = switch (m) { + .sib => try self.addExtra(Mir.MemorySib.encode(m)), + .rip => try self.addExtra(Mir.MemoryRip.encode(m)), + else => unreachable, + }, + } }, + }); +} + fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index { return self.addInst(.{ .tag = .jmp_reloc, @@ -793,18 +832,23 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { switch (air_tags[inst]) { // zig fmt: off - .add => try self.airBinOp(inst, .add), - .addwrap => try self.airBinOp(inst, .addwrap), - .sub => try self.airBinOp(inst, .sub), - .subwrap => try self.airBinOp(inst, .subwrap), - .bool_and => try self.airBinOp(inst, .bool_and), - .bool_or => try self.airBinOp(inst, .bool_or), - .bit_and => try self.airBinOp(inst, .bit_and), - .bit_or => try self.airBinOp(inst, .bit_or), - .xor => try self.airBinOp(inst, .xor), + .not, + => |tag| try self.airUnOp(inst, tag), - .ptr_add => try self.airPtrArithmetic(inst, .ptr_add), - .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub), + .add, + .addwrap, + .sub, + .subwrap, + .bool_and, + .bool_or, + .bit_and, + .bit_or, + .xor, + .min, + .max, + => |tag| try self.airBinOp(inst, tag), + + .ptr_add, .ptr_sub => |tag| try self.airPtrArithmetic(inst, tag), .shr, .shr_exact => try self.airShlShrBinOp(inst), .shl, .shl_exact => try self.airShlShrBinOp(inst), @@ -818,8 +862,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .sub_sat => try self.airSubSat(inst), .mul_sat => try self.airMulSat(inst), .shl_sat => try self.airShlSat(inst), - .min => try self.airMin(inst), - .max => try self.airMax(inst), .slice => try self.airSlice(inst), .sqrt, @@ -867,7 +909,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .breakpoint => try self.airBreakpoint(), .ret_addr => try self.airRetAddr(inst), .frame_addr => try self.airFrameAddress(inst), - .fence => try self.airFence(), + .fence => try self.airFence(inst), .cond_br => try self.airCondBr(inst), .dbg_stmt => try self.airDbgStmt(inst), .fptrunc => try self.airFptrunc(inst), @@ -885,7 +927,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .is_err_ptr => try self.airIsErrPtr(inst), .load => try self.airLoad(inst), .loop => try self.airLoop(inst), - .not => try self.airNot(inst), .ptrtoint => try self.airPtrToInt(inst), .ret => try self.airRet(inst), .ret_load => try self.airRetLoad(inst), @@ -972,10 +1013,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .optional_payload => try self.airOptionalPayload(inst), .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), - .unwrap_errunion_err => try self.airUnwrapErrErr(inst), - .unwrap_errunion_payload => try self.airUnwrapErrPayload(inst), - .unwrap_errunion_err_ptr => try self.airUnwrapErrErrPtr(inst), - .unwrap_errunion_payload_ptr=> try self.airUnwrapErrPayloadPtr(inst), + .unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst), + .unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst), + .unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst), + .unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst), .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), .err_return_trace => try self.airErrReturnTrace(inst), .set_err_return_trace => try self.airSetErrReturnTrace(inst), @@ -1266,7 +1307,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void { } } -pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [count]Register) !void { +pub fn spillRegisters(self: *Self, registers: []const Register) !void { for (registers) |reg| { try self.register_manager.getReg(reg, null); } @@ -1399,8 +1440,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { // when truncating a `u16` to `u5`, for example, those top 3 bits in the result // have to be removed. this only happens if the dst if not a power-of-two size. - const dst_bit_size = dst_ty.bitSize(self.target.*); - if (!math.isPowerOfTwo(dst_bit_size) or dst_bit_size < 8) { + if (self.regExtraBits(dst_ty) > 0) { try self.truncateRegister(dst_ty, reg); } @@ -1414,113 +1454,6 @@ fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ un_op, .none, .none }); } -fn airNot(self: *Self, inst: Air.Inst.Index) !void { - const ty_op = self.air.instructions.items(.data)[inst].ty_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); - } - - const operand_ty = self.air.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); - - const result: MCValue = result: { - switch (operand) { - .dead => unreachable, - .unreach => unreachable, - .eflags => |cc| { - break :result MCValue{ .eflags = cc.negate() }; - }, - else => {}, - } - - const operand_lock: ?RegisterLock = switch (operand) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv: MCValue = blk: { - if (self.reuseOperand(inst, ty_op.operand, 0, operand) and operand.isRegister()) { - break :blk operand; - } - break :blk try self.copyToRegisterWithInstTracking(inst, operand_ty, operand); - }; - const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - const mask = switch (operand_ty.abiSize(self.target.*)) { - 1 => ~@as(u8, 0), - 2 => ~@as(u16, 0), - 4 => ~@as(u32, 0), - 8 => ~@as(u64, 0), - else => unreachable, - }; - try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask }); - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); -} - -fn airMin(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const ty = self.air.typeOfIndex(inst); - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement min for type {}", .{ty.fmtDebug()}); - } - const signedness = ty.intInfo(self.target.*).signedness; - const result: MCValue = result: { - // TODO improve by checking if any operand can be reused. - // TODO audit register allocation - const lhs = try self.resolveInst(bin_op.lhs); - const lhs_lock: ?RegisterLock = switch (lhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const lhs_reg = try self.copyToTmpRegister(ty, lhs); - const lhs_reg_lock = self.register_manager.lockRegAssumeUnused(lhs_reg); - defer self.register_manager.unlockReg(lhs_reg_lock); - - const rhs_mcv = try self.limitImmediateType(bin_op.rhs, i32); - const rhs_lock: ?RegisterLock = switch (rhs_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genBinOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv); - - const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv); - const cc: Condition = switch (signedness) { - .unsigned => .b, - .signed => .l, - }; - try self.asmCmovccRegisterRegister(dst_mcv.register, lhs_reg, cc); - - break :result dst_mcv; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMax(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement max for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airSlice(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -1542,14 +1475,23 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } +fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { + const ty_op = self.air.instructions.items(.data)[inst].ty_op; + + const result = if (self.liveness.isUnused(inst)) + .dead + else + try self.genUnOp(inst, tag, ty_op.operand); + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); +} + fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const result = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); + const result = if (self.liveness.isUnused(inst)) + .dead + else + try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1557,31 +1499,32 @@ fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const result = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); + const result = if (self.liveness.isUnused(inst)) + .dead + else + try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; + const result = result: { + if (self.liveness.isUnused(inst)) break :result .dead; - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } + const tag = self.air.instructions.items(.tag)[inst]; + const ty = self.air.typeOfIndex(inst); - const tag = self.air.instructions.items(.tag)[inst]; - const ty = self.air.typeOfIndex(inst); + if (ty.zigTypeTag() == .Float) { + break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs); + } - try self.spillRegisters(2, .{ .rax, .rdx }); + try self.spillRegisters(&.{ .rax, .rdx }); - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const result = try self.genMulDivBinOp(tag, inst, ty, lhs, rhs); + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + break :result try self.genMulDivBinOp(tag, inst, ty, lhs, rhs); + }; return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1629,7 +1572,7 @@ fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillEflagsIfOccupied(); if (tag == .shl_with_overflow) { - try self.spillRegisters(1, .{.rcx}); + try self.spillRegisters(&.{.rcx}); } const partial: MCValue = switch (tag) { @@ -1756,7 +1699,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillEflagsIfOccupied(); self.eflags_inst = inst; - try self.spillRegisters(2, .{ .rax, .rdx }); + try self.spillRegisters(&.{ .rax, .rdx }); const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -1809,7 +1752,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { break :dst_reg dst_reg; }, .unsigned => { - try self.spillRegisters(2, .{ .rax, .rdx }); + try self.spillRegisters(&.{ .rax, .rdx }); const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -1923,7 +1866,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); } - try self.spillRegisters(1, .{.rcx}); + try self.spillRegisters(&.{.rcx}); const tag = self.air.instructions.items(.tag)[inst]; const lhs = try self.resolveInst(bin_op.lhs); @@ -1947,59 +1890,82 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) !void { fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); - } - - const payload_ty = self.air.typeOfIndex(inst); - const optional_ty = self.air.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); const result: MCValue = result: { - if (!payload_ty.hasRuntimeBits()) break :result MCValue.none; - if (optional_ty.isPtrLikeOptional()) { - if (self.reuseOperand(inst, ty_op.operand, 0, operand)) { - break :result operand; + if (self.liveness.isUnused(inst)) break :result .none; + + const pl_ty = self.air.typeOfIndex(inst); + const opt_mcv = try self.resolveInst(ty_op.operand); + + if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) { + switch (opt_mcv) { + .register => |reg| try self.truncateRegister(pl_ty, reg), + .register_overflow => |ro| try self.truncateRegister(pl_ty, ro.reg), + else => {}, } - break :result try self.copyToRegisterWithInstTracking(inst, payload_ty, operand); + break :result opt_mcv; } - const offset = optional_ty.abiSize(self.target.*) - payload_ty.abiSize(self.target.*); - switch (operand) { - .stack_offset => |off| { - break :result MCValue{ .stack_offset = off - @intCast(i32, offset) }; - }, - .register => { - // TODO reuse the operand - const result = try self.copyToRegisterWithInstTracking(inst, optional_ty, operand); - const shift = @intCast(u8, offset * @sizeOf(usize)); - try self.genShiftBinOpMir(.shr, optional_ty, result.register, .{ .immediate = @intCast(u8, shift) }); - break :result result; - }, - else => return self.fail("TODO implement optional_payload when operand is {}", .{operand}), - } + const pl_mcv = try self.allocRegOrMem(inst, true); + try self.setRegOrMem(pl_ty, pl_mcv, switch (opt_mcv) { + else => opt_mcv, + .register_overflow => |ro| .{ .register = ro.reg }, + }); + break :result pl_mcv; }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement .optional_payload_ptr for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const dst_ty = self.air.typeOfIndex(inst); + const opt_mcv = try self.resolveInst(ty_op.operand); + + break :result if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) + opt_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv); + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement .optional_payload_ptr_set for {}", .{self.target.cpu.arch}); + const result = result: { + const dst_ty = self.air.typeOfIndex(inst); + const src_ty = self.air.typeOf(ty_op.operand); + const opt_ty = src_ty.childType(); + const src_mcv = try self.resolveInst(ty_op.operand); + + if (opt_ty.optionalReprIsPayload()) { + break :result if (self.liveness.isUnused(inst)) + .dead + else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + } + + const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv); + + const pl_ty = dst_ty.childType(); + const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*)); + try self.asmMemoryImmediate( + .mov, + Memory.sib(.byte, .{ .base = dst_mcv.register, .disp = pl_abi_size }), + Immediate.u(1), + ); + break :result if (self.liveness.isUnused(inst)) .dead else dst_mcv; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } -fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void { +fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; if (self.liveness.isUnused(inst)) { return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); @@ -2026,8 +1992,14 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void { }, .register => |reg| { // TODO reuse operand - const lock = self.register_manager.lockRegAssumeUnused(reg); - defer self.register_manager.unlockReg(lock); + self.register_manager.getRegAssumeFree(.rcx, null); + const rcx_lock = + if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; + defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); + + const eu_lock = self.register_manager.lockReg(reg); + defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); + const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); if (err_off > 0) { const shift = @intCast(u6, err_off * 8); @@ -2043,7 +2015,7 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } -fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { +fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; if (self.liveness.isUnused(inst)) { return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); @@ -2075,8 +2047,14 @@ fn genUnwrapErrorUnionPayloadMir( }, .register => |reg| { // TODO reuse operand - const lock = self.register_manager.lockRegAssumeUnused(reg); - defer self.register_manager.unlockReg(lock); + self.register_manager.getRegAssumeFree(.rcx, null); + const rcx_lock = + if (payload_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; + defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); + + const eu_lock = self.register_manager.lockReg(reg); + defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); + const result_reg: Register = if (maybe_inst) |inst| (try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)).register else @@ -2097,31 +2075,118 @@ fn genUnwrapErrorUnionPayloadMir( } // *(E!T) -> E -fn airUnwrapErrErrPtr(self: *Self, inst: Air.Inst.Index) !void { +fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement unwrap error union error ptr for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const src_ty = self.air.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const dst_reg = try self.register_manager.allocReg(inst, gp); + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const eu_ty = src_ty.childType(); + const pl_ty = eu_ty.errorUnionPayload(); + const err_ty = eu_ty.errorUnionSet(); + const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*)); + const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*)); + try self.asmRegisterMemory( + .mov, + registerAlias(dst_reg, err_abi_size), + Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }), + ); + break :result .{ .register = dst_reg }; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } // *(E!T) -> *T -fn airUnwrapErrPayloadPtr(self: *Self, inst: Air.Inst.Index) !void { +fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement unwrap error union payload ptr for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const src_ty = self.air.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const dst_ty = self.air.typeOfIndex(inst); + const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_reg + else + try self.register_manager.allocReg(inst, gp); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const eu_ty = src_ty.childType(); + const pl_ty = eu_ty.errorUnionPayload(); + const pl_off = @intCast(i32, errUnionPayloadOffset(pl_ty, self.target.*)); + const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); + try self.asmRegisterMemory( + .lea, + registerAlias(dst_reg, dst_abi_size), + Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }), + ); + break :result .{ .register = dst_reg }; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement .errunion_payload_ptr_set for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + const src_ty = self.air.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const eu_ty = src_ty.childType(); + const pl_ty = eu_ty.errorUnionPayload(); + const err_ty = eu_ty.errorUnionSet(); + const err_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*)); + const err_abi_size = @intCast(u32, err_ty.abiSize(self.target.*)); + try self.asmMemoryImmediate( + .mov, + Memory.sib(Memory.PtrSize.fromSize(err_abi_size), .{ .base = src_reg, .disp = err_off }), + Immediate.u(0), + ); + + if (self.liveness.isUnused(inst)) break :result .dead; + + const dst_ty = self.air.typeOfIndex(inst); + const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_reg + else + try self.register_manager.allocReg(inst, gp); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const pl_off = @intCast(i32, errUnionErrorOffset(pl_ty, self.target.*)); + const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); + try self.asmRegisterMemory( + .lea, + registerAlias(dst_reg, dst_abi_size), + Memory.sib(.qword, .{ .base = src_reg, .disp = pl_off }), + ); + break :result .{ .register = dst_reg }; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -2145,41 +2210,45 @@ fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void { fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); - } - - const payload_ty = self.air.typeOf(ty_op.operand); const result: MCValue = result: { - if (!payload_ty.hasRuntimeBits()) { - break :result MCValue{ .immediate = 1 }; - } + if (self.liveness.isUnused(inst)) break :result .dead; - const optional_ty = self.air.typeOfIndex(inst); - const operand = try self.resolveInst(ty_op.operand); - const operand_lock: ?RegisterLock = switch (operand) { + const pl_ty = self.air.typeOf(ty_op.operand); + if (!pl_ty.hasRuntimeBits()) break :result .{ .immediate = 1 }; + + const opt_ty = self.air.typeOfIndex(inst); + const pl_mcv = try self.resolveInst(ty_op.operand); + const same_repr = opt_ty.optionalReprIsPayload(); + if (same_repr and self.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv; + + const pl_lock: ?RegisterLock = switch (pl_mcv) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), else => null, }; - defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); + defer if (pl_lock) |lock| self.register_manager.unlockReg(lock); - if (optional_ty.isPtrLikeOptional()) { - // TODO should we check if we can reuse the operand? - if (self.reuseOperand(inst, ty_op.operand, 0, operand)) { - break :result operand; + const opt_mcv = try self.allocRegOrMem(inst, true); + try self.setRegOrMem(pl_ty, opt_mcv, pl_mcv); + + if (!same_repr) { + const pl_abi_size = @intCast(i32, pl_ty.abiSize(self.target.*)); + switch (opt_mcv) { + else => unreachable, + + .register => |opt_reg| try self.asmRegisterImmediate( + .bts, + opt_reg, + Immediate.u(@intCast(u6, pl_abi_size * 8)), + ), + + .stack_offset => |off| try self.asmMemoryImmediate( + .mov, + Memory.sib(.byte, .{ .base = .rsp, .disp = pl_abi_size - off }), + Immediate.u(0), + ), } - break :result try self.copyToRegisterWithInstTracking(inst, payload_ty, operand); } - - const optional_abi_size = @intCast(u32, optional_ty.abiSize(self.target.*)); - const optional_abi_align = optional_ty.abiAlignment(self.target.*); - const payload_abi_size = @intCast(u32, payload_ty.abiSize(self.target.*)); - const offset = optional_abi_size - payload_abi_size; - - const stack_offset = @intCast(i32, try self.allocMem(inst, optional_abi_size, optional_abi_align)); - try self.genSetStack(Type.bool, stack_offset, .{ .immediate = 1 }, .{}); - try self.genSetStack(payload_ty, stack_offset - @intCast(i32, offset), operand, .{}); - break :result MCValue{ .stack_offset = stack_offset }; + break :result opt_mcv; }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -2281,19 +2350,53 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void { fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement ptr_slice_len_ptr for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const src_ty = self.air.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + const src_reg = switch (src_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(src_ty, src_mcv), + }; + const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); + defer self.register_manager.unlockReg(src_lock); + + const dst_ty = self.air.typeOfIndex(inst); + const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_reg + else + try self.register_manager.allocReg(inst, gp); + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); + try self.asmRegisterMemory( + .lea, + registerAlias(dst_reg, dst_abi_size), + Memory.sib(.qword, .{ + .base = src_reg, + .disp = @divExact(self.target.cpu.arch.ptrBitWidth(), 8), + }), + ); + break :result .{ .register = dst_reg }; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement ptr_slice_ptr_ptr for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const dst_ty = self.air.typeOfIndex(inst); + const opt_mcv = try self.resolveInst(ty_op.operand); + + break :result if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) + opt_mcv + else + try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv); + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -2492,10 +2595,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { try self.asmRegisterMemory( .mov, registerAlias(dst_mcv.register, elem_abi_size), - Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{ - .base = dst_mcv.register, - .disp = 0, - }), + Memory.sib(Memory.PtrSize.fromSize(elem_abi_size), .{ .base = dst_mcv.register }), ); break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) }; } @@ -2614,7 +2714,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { }, .register => { const shift: u6 = if (layout.tag_align < layout.payload_align) - @intCast(u6, layout.payload_size * @sizeOf(usize)) + @intCast(u6, layout.payload_size * 8) else 0; const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand); @@ -2632,46 +2732,418 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { fn airClz(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement airClz for {}", .{self.target.cpu.arch}); + const result = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const dst_ty = self.air.typeOfIndex(inst); + const src_ty = self.air.typeOf(ty_op.operand); + + const src_mcv = try self.resolveInst(ty_op.operand); + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, + else => src_mcv, + }; + const mat_src_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_reg = try self.register_manager.allocReg(inst, gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) { + try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv); + const extra_bits = self.regExtraBits(src_ty); + if (extra_bits > 0) { + try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits }); + } + break :result dst_mcv; + } + + const src_bits = src_ty.bitSize(self.target.*); + const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits }); + const width_mcv = MCValue{ .register = width_reg }; + try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv); + + const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2)); + try self.asmCmovccRegisterRegister( + registerAlias(dst_reg, dst_abi_size), + registerAlias(width_reg, dst_abi_size), + .z, + ); + + try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv); + break :result width_mcv; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airCtz(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement airCtz for {}", .{self.target.cpu.arch}); + const result = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const dst_ty = self.air.typeOfIndex(inst); + const src_ty = self.air.typeOf(ty_op.operand); + const src_bits = src_ty.bitSize(self.target.*); + + const src_mcv = try self.resolveInst(ty_op.operand); + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, + else => src_mcv, + }; + const mat_src_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_reg = try self.register_manager.allocReg(inst, gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) { + const extra_bits = self.regExtraBits(src_ty); + const masked_mcv = if (extra_bits > 0) masked: { + const mask_mcv = MCValue{ + .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) << @intCast(u6, src_bits), + }; + const tmp_mcv = tmp: { + if (src_mcv.isImmediate() or self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + break :tmp src_mcv; + } + try self.genSetReg(src_ty, dst_reg, src_mcv); + break :tmp dst_mcv; + }; + try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv); + break :masked tmp_mcv; + } else mat_src_mcv; + try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv); + break :result dst_mcv; + } + + const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits }); + try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv); + + const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2); + try self.asmCmovccRegisterRegister( + registerAlias(dst_reg, abi_size), + registerAlias(width_reg, abi_size), + .z, + ); + + break :result dst_mcv; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airPopcount(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement airPopcount for {}", .{self.target.cpu.arch}); + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const src_ty = self.air.typeOf(ty_op.operand); + const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); + const src_mcv = try self.resolveInst(ty_op.operand); + + if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) { + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }, + else => src_mcv, + }; + const mat_src_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv: MCValue = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + .{ .register = try self.register_manager.allocReg(inst, gp) }; + + const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16; + try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv); + break :result dst_mcv; + } + + const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8); + const imm_0_1 = Immediate.u(mask / 0b1_1); + const imm_00_11 = Immediate.u(mask / 0b01_01); + const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); + const imm_0000_0001 = Immediate.u(mask / 0b1111_1111); + + const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv.register + else + try self.copyToTmpRegister(src_ty, src_mcv); + const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); + defer self.register_manager.unlockReg(tmp_lock); + + const dst_reg = try self.register_manager.allocReg(inst, gp); + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + { + const dst = registerAlias(dst_reg, src_abi_size); + const tmp = registerAlias(tmp_reg, src_abi_size); + const imm = if (src_abi_size > 4) + try self.register_manager.allocReg(null, gp) + else + undefined; + + // tmp = operand + try self.asmRegisterRegister(.mov, dst, tmp); + // dst = operand + try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1)); + // tmp = operand >> 1 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.mov, imm, imm_0_1); + try self.asmRegisterRegister(.@"and", tmp, imm); + } else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1); + // tmp = (operand >> 1) & 0x55...55 + try self.asmRegisterRegister(.sub, dst, tmp); + // dst = temp1 = operand - ((operand >> 1) & 0x55...55) + try self.asmRegisterRegister(.mov, tmp, dst); + // tmp = temp1 + try self.asmRegisterImmediate(.shr, dst, Immediate.u(2)); + // dst = temp1 >> 2 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.mov, imm, imm_00_11); + try self.asmRegisterRegister(.@"and", tmp, imm); + try self.asmRegisterRegister(.@"and", dst, imm); + } else { + try self.asmRegisterImmediate(.@"and", tmp, imm_00_11); + try self.asmRegisterImmediate(.@"and", dst, imm_00_11); + } + // tmp = temp1 & 0x33...33 + // dst = (temp1 >> 2) & 0x33...33 + try self.asmRegisterRegister(.add, tmp, dst); + // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33) + try self.asmRegisterRegister(.mov, dst, tmp); + // dst = temp2 + try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4)); + // tmp = temp2 >> 4 + try self.asmRegisterRegister(.add, dst, tmp); + // dst = temp2 + (temp2 >> 4) + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.mov, imm, imm_0000_1111); + try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001); + try self.asmRegisterRegister(.@"and", dst, imm); + try self.asmRegisterRegister(.imul, dst, tmp); + } else { + try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111); + if (src_abi_size > 1) { + try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001); + } + } + // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f + // dst = temp3 * 0x01...01 + if (src_abi_size > 1) { + try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8)); + } + // dst = (temp3 * 0x01...01) >> (bits - 8) + } + break :result .{ .register = dst_reg }; + }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } +fn byteSwap(self: *Self, inst: Air.Inst.Index, src_ty: Type, src_mcv: MCValue, mem_ok: bool) !MCValue { + const ty_op = self.air.instructions.items(.data)[inst].ty_op; + + const src_bits = self.regBitSize(src_ty); + const src_lock = switch (src_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + switch (src_bits) { + else => unreachable, + 8 => return if ((mem_ok or src_mcv.isRegister()) and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv), + 16 => if ((mem_ok or src_mcv.isRegister()) and + self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) + { + try self.genBinOpMir(.rol, src_ty, src_mcv, .{ .immediate = 8 }); + return src_mcv; + }, + 32, 64 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + try self.genUnOpMir(.bswap, src_ty, src_mcv); + return src_mcv; + }, + } + + if (src_mcv.isRegister()) { + const dst_mcv: MCValue = if (mem_ok) + try self.allocRegOrMem(inst, true) + else + .{ .register = try self.register_manager.allocReg(inst, gp) }; + if (dst_mcv.isRegister()) { + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register); + defer self.register_manager.unlockReg(dst_lock); + + try self.genSetReg(src_ty, dst_mcv.register, src_mcv); + switch (src_bits) { + else => unreachable, + 16 => try self.genBinOpMir(.rol, src_ty, dst_mcv, .{ .immediate = 8 }), + 32, 64 => try self.genUnOpMir(.bswap, src_ty, dst_mcv), + } + } else try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv); + return dst_mcv; + } + + const dst_reg = try self.register_manager.allocReg(inst, gp); + const dst_mcv = MCValue{ .register = dst_reg }; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + try self.genBinOpMir(.movbe, src_ty, dst_mcv, src_mcv); + return dst_mcv; +} + fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement airByteSwap for {}", .{self.target.cpu.arch}); + const result = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const src_ty = self.air.typeOf(ty_op.operand); + const src_mcv = try self.resolveInst(ty_op.operand); + + const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, true); + switch (self.regExtraBits(src_ty)) { + 0 => {}, + else => |extra| try self.genBinOpMir( + if (src_ty.isSignedInt()) .sar else .shr, + src_ty, + dst_mcv, + .{ .immediate = extra }, + ), + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement airBitReverse for {}", .{self.target.cpu.arch}); + const result = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const src_ty = self.air.typeOf(ty_op.operand); + const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*)); + const src_mcv = try self.resolveInst(ty_op.operand); + + const dst_mcv = try self.byteSwap(inst, src_ty, src_mcv, false); + const dst_reg = dst_mcv.register; + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + + const tmp_reg = try self.register_manager.allocReg(null, gp); + const tmp_lock = self.register_manager.lockReg(tmp_reg); + defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock); + + { + const dst = registerAlias(dst_reg, src_abi_size); + const tmp = registerAlias(tmp_reg, src_abi_size); + const imm = if (src_abi_size > 4) + try self.register_manager.allocReg(null, gp) + else + undefined; + + const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8); + const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); + const imm_00_11 = Immediate.u(mask / 0b01_01); + const imm_0_1 = Immediate.u(mask / 0b1_1); + + // dst = temp1 = bswap(operand) + try self.asmRegisterRegister(.mov, tmp, dst); + // tmp = temp1 + try self.asmRegisterImmediate(.shr, dst, Immediate.u(4)); + // dst = temp1 >> 4 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.mov, imm, imm_0000_1111); + try self.asmRegisterRegister(.@"and", tmp, imm); + try self.asmRegisterRegister(.@"and", dst, imm); + } else { + try self.asmRegisterImmediate(.@"and", tmp, imm_0000_1111); + try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111); + } + // tmp = temp1 & 0x0F...0F + // dst = (temp1 >> 4) & 0x0F...0F + try self.asmRegisterImmediate(.shl, tmp, Immediate.u(4)); + // tmp = (temp1 & 0x0F...0F) << 4 + try self.asmRegisterRegister(.@"or", dst, tmp); + // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4) + try self.asmRegisterRegister(.mov, tmp, dst); + // tmp = temp2 + try self.asmRegisterImmediate(.shr, dst, Immediate.u(2)); + // dst = temp2 >> 2 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.mov, imm, imm_00_11); + try self.asmRegisterRegister(.@"and", tmp, imm); + try self.asmRegisterRegister(.@"and", dst, imm); + } else { + try self.asmRegisterImmediate(.@"and", tmp, imm_00_11); + try self.asmRegisterImmediate(.@"and", dst, imm_00_11); + } + // tmp = temp2 & 0x33...33 + // dst = (temp2 >> 2) & 0x33...33 + try self.asmRegisterMemory( + .lea, + if (src_abi_size > 4) tmp.to64() else tmp.to32(), + Memory.sib(.qword, .{ + .base = dst.to64(), + .scale_index = .{ .index = tmp.to64(), .scale = 1 << 2 }, + }), + ); + // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2) + try self.asmRegisterRegister(.mov, dst, tmp); + // dst = temp3 + try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1)); + // tmp = temp3 >> 1 + if (src_abi_size > 4) { + try self.asmRegisterImmediate(.mov, imm, imm_0_1); + try self.asmRegisterRegister(.@"and", dst, imm); + try self.asmRegisterRegister(.@"and", tmp, imm); + } else { + try self.asmRegisterImmediate(.@"and", dst, imm_0_1); + try self.asmRegisterImmediate(.@"and", tmp, imm_0_1); + } + // dst = temp3 & 0x55...55 + // tmp = (temp3 >> 1) & 0x55...55 + try self.asmRegisterMemory( + .lea, + if (src_abi_size > 4) dst.to64() else dst.to32(), + Memory.sib(.qword, .{ + .base = tmp.to64(), + .scale_index = .{ .index = dst.to64(), .scale = 1 << 1 }, + }), + ); + // dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1) + } + + switch (self.regExtraBits(src_ty)) { + 0 => {}, + else => |extra| try self.genBinOpMir( + if (src_ty.isSignedInt()) .sar else .shr, + src_ty, + dst_mcv, + .{ .immediate = extra }, + ), + } + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -2753,7 +3225,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo try self.asmRegisterMemory( .mov, registerAlias(dst_reg, abi_size), - Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg, .disp = 0 }), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg }), ); }, .stack_offset => |off| { @@ -2865,8 +3337,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .none => unreachable, .dead => unreachable, .unreach => unreachable, - .eflags => unreachable, + .eflags => |cc| { + try self.asmSetccMemory(Memory.sib( + Memory.PtrSize.fromSize(abi_size), + .{ .base = reg.to64() }, + ), cc); + }, .undef => { + if (!self.wantSafety()) return; // The already existing value will do just fine. switch (abi_size) { 1 => try self.store(ptr, .{ .immediate = 0xaa }, ptr_ty, value_ty), 2 => try self.store(ptr, .{ .immediate = 0xaaaa }, ptr_ty, value_ty), @@ -2882,10 +3360,10 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type Immediate.s(@intCast(i32, @bitCast(i64, imm))) else Immediate.u(@truncate(u32, imm)); - try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ - .base = reg.to64(), - .disp = 0, - }), immediate); + try self.asmMemoryImmediate(.mov, Memory.sib( + Memory.PtrSize.fromSize(abi_size), + .{ .base = reg.to64() }, + ), immediate); }, 8 => { // TODO: optimization: if the imm is only using the lower @@ -2957,10 +3435,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr); // To get the actual address of the value we want to modify we have to go through the GOT - try self.asmRegisterMemory(.mov, addr_reg.to64(), Memory.sib(.qword, .{ - .base = addr_reg.to64(), - .disp = 0, - })); + try self.asmRegisterMemory( + .mov, + addr_reg.to64(), + Memory.sib(.qword, .{ .base = addr_reg.to64() }), + ); const new_ptr = MCValue{ .register = addr_reg.to64() }; @@ -2982,10 +3461,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type return self.fail("TODO imm64 would get incorrectly sign extended", .{}); } } - try self.asmMemoryImmediate(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ - .base = addr_reg.to64(), - .disp = 0, - }), Immediate.u(@intCast(u32, imm))); + try self.asmMemoryImmediate( + .mov, + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg.to64() }), + Immediate.u(@intCast(u32, imm)), + ); }, .register => { return self.store(new_ptr, value, ptr_ty, value_ty); @@ -2997,10 +3477,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type defer self.register_manager.unlockReg(tmp_reg_lock); try self.loadMemPtrIntoRegister(tmp_reg, value_ty, value); - try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(.qword, .{ - .base = tmp_reg, - .disp = 0, - })); + try self.asmRegisterMemory( + .mov, + tmp_reg, + Memory.sib(.qword, .{ .base = tmp_reg }), + ); return self.store(new_ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); } @@ -3152,7 +3633,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock); // Shift by struct_field_offset. - const shift = @intCast(u8, struct_field_offset * @sizeOf(usize)); + const shift = @intCast(u8, struct_field_offset * 8); try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift }); // Mask with reg.bitSize() - struct_field_size @@ -3212,6 +3693,107 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } +fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue { + const src_ty = self.air.typeOf(src_air); + const src_mcv = try self.resolveInst(src_air); + if (src_ty.zigTypeTag() == .Vector) { + return self.fail("TODO implement genBinOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)}); + } + if (src_ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement genBinOp for {}", .{src_ty.fmt(self.bin_file.options.module.?)}); + } + + switch (src_mcv) { + .eflags => |cc| switch (tag) { + .not => return .{ .eflags = cc.negate() }, + else => {}, + }, + else => {}, + } + + const src_lock = switch (src_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv: MCValue = if (maybe_inst) |inst| + if (self.reuseOperand(inst, src_air, 0, src_mcv)) + src_mcv + else + try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv) + else + .{ .register = try self.copyToTmpRegister(src_ty, src_mcv) }; + const dst_lock = switch (dst_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + switch (tag) { + .not => { + const int_info = if (src_ty.tag() == .bool) + std.builtin.Type.Int{ .signedness = .unsigned, .bits = 1 } + else + src_ty.intInfo(self.target.*); + const extra_bits = self.regExtraBits(src_ty); + if (int_info.signedness == .unsigned and extra_bits > 0) { + const mask = (@as(u64, 1) << @intCast(u6, src_ty.bitSize(self.target.*))) - 1; + try self.genBinOpMir(.xor, src_ty, dst_mcv, .{ .immediate = mask }); + } else try self.genUnOpMir(.not, src_ty, dst_mcv); + }, + + .neg => try self.genUnOpMir(.neg, src_ty, dst_mcv), + + else => unreachable, + } + return dst_mcv; +} + +fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue) !void { + const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); + switch (dst_mcv) { + .none => unreachable, + .undef => unreachable, + .dead, .unreach, .immediate => unreachable, + .eflags => unreachable, + .register_overflow => unreachable, + .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), + .ptr_stack_offset, .stack_offset => |off| { + if (off > math.maxInt(i32)) { + return self.fail("stack offset too large", .{}); + } + if (abi_size > 8) { + return self.fail("TODO implement {} for stack dst with large ABI", .{mir_tag}); + } + + try self.asmMemory(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ + .base = .rbp, + .disp = -off, + })); + }, + .memory, .linker_load => { + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + + try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv); + + // To get the actual address of the value we want to modify we have to go through the GOT + try self.asmRegisterMemory( + .mov, + addr_reg, + Memory.sib(.qword, .{ .base = addr_reg }), + ); + + try self.asmMemory( + mir_tag, + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }), + ); + }, + } +} + /// Clobbers .rcx for non-immediate shift value. fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void { assert(reg.to64() != .rcx); @@ -3233,8 +3815,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi }, else => {}, } - assert(self.register_manager.isRegFree(.rcx)); - try self.register_manager.getReg(.rcx, null); + self.register_manager.getRegAssumeFree(.rcx, null); try self.genSetReg(Type.u8, .rcx, shift); } @@ -3274,8 +3855,7 @@ fn genShiftBinOp( }; defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - assert(self.register_manager.isRegFree(.rcx)); - try self.register_manager.getReg(.rcx, null); + self.register_manager.getRegAssumeFree(.rcx, null); const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); defer self.register_manager.unlockReg(rcx_lock); @@ -3325,10 +3905,10 @@ fn genMulDivBinOp( rhs: MCValue, ) !MCValue { if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) { - return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()}); + return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()}); } if (ty.abiSize(self.target.*) > 8) { - return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()}); + return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()}); } if (tag == .div_float) { return self.fail("TODO implement genMulDivBinOp for div_float", .{}); @@ -3486,10 +4066,21 @@ fn genBinOp( const lhs_ty = self.air.typeOf(lhs_air); const rhs_ty = self.air.typeOf(rhs_air); if (lhs_ty.zigTypeTag() == .Vector) { - return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); + return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)}); } if (lhs_ty.abiSize(self.target.*) > 8) { - return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); + return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmt(self.bin_file.options.module.?)}); + } + + switch (lhs) { + .immediate => |imm| switch (imm) { + 0 => switch (tag) { + .sub, .subwrap => return self.genUnOp(maybe_inst, .neg, rhs_air), + else => {}, + }, + else => {}, + }, + else => {}, } const is_commutative: bool = switch (tag) { @@ -3500,6 +4091,8 @@ fn genBinOp( .bool_and, .bit_and, .xor, + .min, + .max, => true, else => false, @@ -3512,7 +4105,7 @@ fn genBinOp( defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); const rhs_lock: ?RegisterLock = switch (rhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + .register => |reg| self.register_manager.lockReg(reg), else => null, }; defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); @@ -3520,10 +4113,10 @@ fn genBinOp( var flipped: bool = false; const dst_mcv: MCValue = blk: { if (maybe_inst) |inst| { - if (self.reuseOperand(inst, lhs_air, 0, lhs) and lhs.isRegister()) { + if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) { break :blk lhs; } - if (is_commutative and self.reuseOperand(inst, rhs_air, 1, rhs) and rhs.isRegister()) { + if (is_commutative and rhs.isRegister() and self.reuseOperand(inst, rhs_air, 1, rhs)) { flipped = true; break :blk rhs; } @@ -3551,11 +4144,56 @@ fn genBinOp( switch (tag) { .add, .addwrap, - => try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv), + => try self.genBinOpMir(switch (lhs_ty.tag()) { + else => .add, + .f32 => .addss, + .f64 => .addsd, + }, lhs_ty, dst_mcv, src_mcv), .sub, .subwrap, - => try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv), + => try self.genBinOpMir(switch (lhs_ty.tag()) { + else => .sub, + .f32 => .subss, + .f64 => .subsd, + }, lhs_ty, dst_mcv, src_mcv), + + .mul => try self.genBinOpMir(switch (lhs_ty.tag()) { + .f32 => .mulss, + .f64 => .mulsd, + else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }), + }, lhs_ty, dst_mcv, src_mcv), + + .div_float, + .div_exact, + => try self.genBinOpMir(switch (lhs_ty.tag()) { + .f32 => .divss, + .f64 => .divsd, + else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }), + }, lhs_ty, dst_mcv, src_mcv), + + .div_trunc, + .div_floor, + => { + try self.genBinOpMir(switch (lhs_ty.tag()) { + .f32 => .divss, + .f64 => .divsd, + else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }), + }, lhs_ty, dst_mcv, src_mcv); + if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) { + const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*)); + const dst_alias = registerAlias(dst_mcv.register, abi_size); + try self.asmRegisterRegisterImmediate(switch (lhs_ty.tag()) { + .f32 => .roundss, + .f64 => .roundsd, + else => unreachable, + }, dst_alias, dst_alias, Immediate.u(switch (tag) { + .div_trunc => 0b1_0_11, + .div_floor => 0b1_0_01, + else => unreachable, + })); + } else return self.fail("TODO implement round without sse4_1", .{}); + }, .ptr_add, .ptr_sub, @@ -3580,6 +4218,113 @@ fn genBinOp( .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv), + .min, + .max, + => switch (lhs_ty.zigTypeTag()) { + .Int => { + const mat_src_mcv = switch (src_mcv) { + .immediate => MCValue{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) }, + else => src_mcv, + }; + const mat_mcv_lock = switch (mat_src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genBinOpMir(.cmp, lhs_ty, dst_mcv, mat_src_mcv); + + const int_info = lhs_ty.intInfo(self.target.*); + const cc: Condition = switch (int_info.signedness) { + .unsigned => switch (tag) { + .min => .a, + .max => .b, + else => unreachable, + }, + .signed => switch (tag) { + .min => .g, + .max => .l, + else => unreachable, + }, + }; + + const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*)); + switch (dst_mcv) { + .none, + .undef, + .dead, + .unreach, + .immediate, + .eflags, + .register_overflow, + .stack_offset, + .ptr_stack_offset, + .memory, + .linker_load, + => unreachable, + .register => |dst_reg| switch (mat_src_mcv) { + .none, + .undef, + .dead, + .unreach, + .immediate, + .eflags, + .register_overflow, + .ptr_stack_offset, + => unreachable, + .register => |src_reg| try self.asmCmovccRegisterRegister( + registerAlias(dst_reg, abi_size), + registerAlias(src_reg, abi_size), + cc, + ), + .stack_offset => |off| try self.asmCmovccRegisterMemory( + registerAlias(dst_reg, abi_size), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ + .base = .rbp, + .disp = -off, + }), + cc, + ), + .memory, .linker_load => { + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + + try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv); + + // To get the actual address of the value we want to modify we + // we have to go through the GOT + try self.asmRegisterMemory( + .mov, + addr_reg, + Memory.sib(.qword, .{ .base = addr_reg }), + ); + + try self.asmCmovccRegisterMemory( + registerAlias(dst_reg, abi_size), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }), + cc, + ); + }, + }, + } + }, + .Float => try self.genBinOpMir(switch (lhs_ty.tag()) { + .f32 => switch (tag) { + .min => .minss, + .max => .maxss, + else => unreachable, + }, + .f64 => switch (tag) { + .min => .minsd, + .max => .maxsd, + else => unreachable, + }, + else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }), + }, lhs_ty, dst_mcv, src_mcv), + else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }), + }, + else => unreachable, } return dst_mcv; @@ -3609,29 +4354,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu .register => |src_reg| switch (dst_ty.zigTypeTag()) { .Float => { if (intrinsicsAllowed(self.target.*, dst_ty)) { - const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) { - .f32 => switch (mir_tag) { - .add => .addss, - .cmp => .ucomiss, - else => return self.fail( - "TODO genBinOpMir for f32 register-register with MIR tag {}", - .{mir_tag}, - ), - }, - .f64 => switch (mir_tag) { - .add => .addsd, - .cmp => .ucomisd, - else => return self.fail( - "TODO genBinOpMir for f64 register-register with MIR tag {}", - .{mir_tag}, - ), - }, - else => return self.fail( - "TODO genBinOpMir for float register-register and type {}", - .{dst_ty.fmtDebug()}, - ), - }; - return self.asmRegisterRegister(actual_tag, dst_reg.to128(), src_reg.to128()); + return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128()); } return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{}); @@ -3643,16 +4366,15 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu ), }, .immediate => |imm| { - switch (abi_size) { - 0 => unreachable, - 1...4 => { + switch (self.regBitSize(dst_ty)) { + 8, 16, 32 => { try self.asmRegisterImmediate( mir_tag, registerAlias(dst_reg, abi_size), Immediate.u(@intCast(u32, imm)), ); }, - 5...8 => { + 64 => { if (math.cast(i32, @bitCast(i64, imm))) |small| { try self.asmRegisterImmediate( mir_tag, @@ -3660,13 +4382,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu Immediate.s(small), ); } else { - const tmp_reg = try self.register_manager.allocReg(null, gp); - const tmp_alias = registerAlias(tmp_reg, abi_size); - try self.asmRegisterImmediate(.mov, tmp_alias, Immediate.u(imm)); try self.asmRegisterRegister( mir_tag, registerAlias(dst_reg, abi_size), - tmp_alias, + registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size), ); } }, @@ -3716,11 +4435,43 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }), registerAlias(src_reg, abi_size)); }, .immediate => |imm| { - // TODO - try self.asmMemoryImmediate(mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ - .base = .rbp, - .disp = -off, - }), Immediate.u(@intCast(u32, imm))); + switch (self.regBitSize(dst_ty)) { + 8, 16, 32 => { + try self.asmMemoryImmediate( + mir_tag, + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ + .base = .rbp, + .disp = -off, + }), + if (math.cast(i32, @bitCast(i64, imm))) |small| + Immediate.s(small) + else + Immediate.u(@intCast(u32, imm)), + ); + }, + 64 => { + if (math.cast(i32, @bitCast(i64, imm))) |small| { + try self.asmMemoryImmediate( + mir_tag, + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ + .base = .rbp, + .disp = -off, + }), + Immediate.s(small), + ); + } else { + try self.asmMemoryRegister( + mir_tag, + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ + .base = .rbp, + .disp = -off, + }), + registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size), + ); + } + }, + else => return self.fail("TODO getBinOpMir implement large immediate ABI", .{}), + } }, .memory, .stack_offset, @@ -3952,18 +4703,33 @@ fn airBreakpoint(self: *Self) !void { } fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void { - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airRetAddr for x86_64", .{}); + const result = if (self.liveness.isUnused(inst)) .dead else result: { + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.setRegOrMem(Type.usize, dst_mcv, .{ + .stack_offset = -@as(i32, @divExact(self.target.cpu.arch.ptrBitWidth(), 8)), + }); + break :result dst_mcv; + }; return self.finishAir(inst, result, .{ .none, .none, .none }); } fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void { - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airFrameAddress for x86_64", .{}); + const result = if (self.liveness.isUnused(inst)) .dead else result: { + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.setRegOrMem(Type.usize, dst_mcv, .{ .register = .rbp }); + break :result dst_mcv; + }; return self.finishAir(inst, result, .{ .none, .none, .none }); } -fn airFence(self: *Self) !void { - return self.fail("TODO implement fence() for {}", .{self.target.cpu.arch}); - //return self.finishAirBookkeeping(); +fn airFence(self: *Self, inst: Air.Inst.Index) !void { + const order = self.air.instructions.items(.data)[inst].fence; + switch (order) { + .Unordered, .Monotonic => unreachable, + .Acquire, .Release, .AcqRel => {}, + .SeqCst => try self.asmOpOnly(.mfence), + } + return self.finishAirBookkeeping(); } fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void { @@ -3984,11 +4750,40 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier defer info.deinit(self); try self.spillEflagsIfOccupied(); + try self.spillRegisters(abi.getCallerPreservedRegs(self.target.*)); - for (abi.getCallerPreservedRegs(self.target.*)) |reg| { - try self.register_manager.getReg(reg, null); + // set stack arguments first because this can clobber registers + // also clobber spill arguments as we go + if (info.return_value == .stack_offset) { + try self.spillRegisters(&.{abi.getCAbiIntParamRegs(self.target.*)[0]}); + } + for (args, info.args) |arg, mc_arg| { + const arg_ty = self.air.typeOf(arg); + const arg_mcv = try self.resolveInst(arg); + // Here we do not use setRegOrMem even though the logic is similar, because + // the function call will move the stack pointer, so the offsets are different. + switch (mc_arg) { + .none => {}, + .register => |reg| try self.spillRegisters(&.{reg}), + .stack_offset => |off| { + // TODO rewrite using `genSetStack` + try self.genSetStackArg(arg_ty, off, arg_mcv); + }, + .ptr_stack_offset => { + return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); + }, + .undef => unreachable, + .immediate => unreachable, + .unreach => unreachable, + .dead => unreachable, + .memory => unreachable, + .linker_load => unreachable, + .eflags => unreachable, + .register_overflow => unreachable, + } } + // now we are free to set register arguments const ret_reg_lock: ?RegisterLock = blk: { if (info.return_value == .stack_offset) { const ret_ty = fn_ty.fnReturnType(); @@ -3998,7 +4793,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier log.debug("airCall: return value on stack at offset {}", .{stack_offset}); const ret_reg = abi.getCAbiIntParamRegs(self.target.*)[0]; - try self.register_manager.getReg(ret_reg, null); try self.genSetReg(Type.usize, ret_reg, .{ .ptr_stack_offset = stack_offset }); const ret_reg_lock = self.register_manager.lockRegAssumeUnused(ret_reg); @@ -4010,25 +4804,12 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier }; defer if (ret_reg_lock) |lock| self.register_manager.unlockReg(lock); - for (args, info.args) |arg, info_arg| { - const mc_arg = info_arg; + for (args, info.args) |arg, mc_arg| { const arg_ty = self.air.typeOf(arg); const arg_mcv = try self.resolveInst(arg); - // Here we do not use setRegOrMem even though the logic is similar, because - // the function call will move the stack pointer, so the offsets are different. switch (mc_arg) { - .none => continue, - .register => |reg| { - try self.register_manager.getReg(reg, null); - try self.genSetReg(arg_ty, reg, arg_mcv); - }, - .stack_offset => |off| { - // TODO rewrite using `genSetStack` - try self.genSetStackArg(arg_ty, off, arg_mcv); - }, - .ptr_stack_offset => { - return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); - }, + .none, .stack_offset, .ptr_stack_offset => {}, + .register => |reg| try self.genSetReg(arg_ty, reg, arg_mcv), .undef => unreachable, .immediate => unreachable, .unreach => unreachable, @@ -4293,7 +5074,11 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { }; defer if (src_lock) |lock| self.register_manager.unlockReg(lock); - try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv); + try self.genBinOpMir(switch (ty.tag()) { + else => .cmp, + .f32 => .ucomiss, + .f64 => .ucomisd, + }, ty, dst_mcv, src_mcv); break :result switch (signedness) { .signed => MCValue{ .eflags = Condition.fromCompareOperatorSigned(op) }, @@ -4510,25 +5295,113 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, .unreach, .{ .none, .none, .none }); } -fn isNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue { +fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue { + switch (opt_mcv) { + .register_overflow => |ro| return .{ .eflags = ro.eflags.negate() }, + else => {}, + } + try self.spillEflagsIfOccupied(); self.eflags_inst = inst; - const cmp_ty: Type = if (!ty.isPtrLikeOptional()) blk: { - var buf: Type.Payload.ElemType = undefined; - const payload_ty = ty.optionalChild(&buf); - break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else ty; - } else ty; + var pl_buf: Type.Payload.ElemType = undefined; + const pl_ty = opt_ty.optionalChild(&pl_buf); - try self.genBinOpMir(.cmp, cmp_ty, operand, MCValue{ .immediate = 0 }); + var ptr_buf: Type.SlicePtrFieldTypeBuffer = undefined; + const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload()) + .{ .off = 0, .ty = if (pl_ty.isSlice()) pl_ty.slicePtrFieldType(&ptr_buf) else pl_ty } + else + .{ .off = @intCast(i32, pl_ty.abiSize(self.target.*)), .ty = Type.bool }; - return MCValue{ .eflags = .e }; + switch (opt_mcv) { + .none, + .unreach, + .dead, + .undef, + .immediate, + .register_overflow, + .ptr_stack_offset, + .eflags, + => unreachable, + + .register => |opt_reg| { + if (some_info.off == 0) { + const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*)); + const alias_reg = registerAlias(opt_reg, some_abi_size); + assert(some_abi_size * 8 == alias_reg.bitSize()); + try self.asmRegisterRegister(.@"test", alias_reg, alias_reg); + return .{ .eflags = .z }; + } + assert(some_info.ty.tag() == .bool); + const opt_abi_size = @intCast(u32, opt_ty.abiSize(self.target.*)); + try self.asmRegisterImmediate( + .bt, + registerAlias(opt_reg, opt_abi_size), + Immediate.u(@intCast(u6, some_info.off * 8)), + ); + return .{ .eflags = .nc }; + }, + + .memory, .linker_load => { + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + + try self.loadMemPtrIntoRegister(addr_reg, Type.usize, opt_mcv); + + // To get the actual address of the value we want to modify we have to go through the GOT + try self.asmRegisterMemory( + .mov, + addr_reg, + Memory.sib(.qword, .{ .base = addr_reg }), + ); + + const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*)); + try self.asmMemoryImmediate(.cmp, Memory.sib( + Memory.PtrSize.fromSize(some_abi_size), + .{ .base = addr_reg, .disp = some_info.off }, + ), Immediate.u(0)); + return .{ .eflags = .e }; + }, + + .stack_offset => |off| { + const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*)); + try self.asmMemoryImmediate(.cmp, Memory.sib( + Memory.PtrSize.fromSize(some_abi_size), + .{ .base = .rbp, .disp = some_info.off - off }, + ), Immediate.u(0)); + return .{ .eflags = .e }; + }, + } } -fn isNonNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue { - const is_null_res = try self.isNull(inst, ty, operand); - assert(is_null_res.eflags == .e); - return MCValue{ .eflags = is_null_res.eflags.negate() }; +fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue { + try self.spillEflagsIfOccupied(); + self.eflags_inst = inst; + + const opt_ty = ptr_ty.childType(); + var pl_buf: Type.Payload.ElemType = undefined; + const pl_ty = opt_ty.optionalChild(&pl_buf); + + var ptr_buf: Type.SlicePtrFieldTypeBuffer = undefined; + const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload()) + .{ .off = 0, .ty = if (pl_ty.isSlice()) pl_ty.slicePtrFieldType(&ptr_buf) else pl_ty } + else + .{ .off = @intCast(i32, pl_ty.abiSize(self.target.*)), .ty = Type.bool }; + + const ptr_reg = switch (ptr_mcv) { + .register => |reg| reg, + else => try self.copyToTmpRegister(ptr_ty, ptr_mcv), + }; + const ptr_lock = self.register_manager.lockReg(ptr_reg); + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*)); + try self.asmMemoryImmediate(.cmp, Memory.sib( + Memory.PtrSize.fromSize(some_abi_size), + .{ .base = ptr_reg, .disp = some_info.off }, + ), Immediate.u(0)); + return .{ .eflags = .e }; } fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) !MCValue { @@ -4550,8 +5423,13 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, ty: Type, operand: MCValue) ! try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 }); }, .register => |reg| { - const maybe_lock = self.register_manager.lockReg(reg); - defer if (maybe_lock) |lock| self.register_manager.unlockReg(lock); + self.register_manager.getRegAssumeFree(.rcx, null); + const rcx_lock = if (err_off > 0) self.register_manager.lockRegAssumeUnused(.rcx) else null; + defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock); + + const eu_lock = self.register_manager.lockReg(reg); + defer if (eu_lock) |lock| self.register_manager.unlockReg(lock); + const tmp_reg = try self.copyToTmpRegister(ty, operand); if (err_off > 0) { const shift = @intCast(u6, err_off * 8); @@ -4594,29 +5472,11 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void { fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void { const un_op = self.air.instructions.items(.data)[inst].un_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ un_op, .none, .none }); - } - - const operand_ptr = try self.resolveInst(un_op); - const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(un_op); + const ty = self.air.typeOf(un_op); + break :result try self.isNullPtr(inst, ty, operand); }; - defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock); - - const ptr_ty = self.air.typeOf(un_op); - const elem_ty = ptr_ty.childType(); - const operand = if (elem_ty.isPtrLikeOptional() and self.reuseOperand(inst, un_op, 0, operand_ptr)) - // The MCValue that holds the pointer can be re-used as the value. - operand_ptr - else - try self.allocTempRegOrMem(elem_ty, true); - try self.load(operand, operand_ptr, ptr_ty); - - const result = try self.isNull(inst, elem_ty, operand); - return self.finishAir(inst, result, .{ un_op, .none, .none }); } @@ -4625,36 +5485,24 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void { const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { const operand = try self.resolveInst(un_op); const ty = self.air.typeOf(un_op); - break :result try self.isNonNull(inst, ty, operand); + break :result switch (try self.isNull(inst, ty, operand)) { + .eflags => |cc| .{ .eflags = cc.negate() }, + else => unreachable, + }; }; return self.finishAir(inst, result, .{ un_op, .none, .none }); } fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void { const un_op = self.air.instructions.items(.data)[inst].un_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ un_op, .none, .none }); - } - - const operand_ptr = try self.resolveInst(un_op); - const operand_ptr_lock: ?RegisterLock = switch (operand_ptr) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { + const operand = try self.resolveInst(un_op); + const ty = self.air.typeOf(un_op); + break :result switch (try self.isNullPtr(inst, ty, operand)) { + .eflags => |cc| .{ .eflags = cc.negate() }, + else => unreachable, + }; }; - defer if (operand_ptr_lock) |lock| self.register_manager.unlockReg(lock); - - const ptr_ty = self.air.typeOf(un_op); - const elem_ty = ptr_ty.childType(); - const operand = if (elem_ty.isPtrLikeOptional() and self.reuseOperand(inst, un_op, 0, operand_ptr)) - // The MCValue that holds the pointer can be re-used as the value. - operand_ptr - else - try self.allocTempRegOrMem(elem_ty, true); - try self.load(operand, operand_ptr, ptr_ty); - - const result = try self.isNonNull(inst, ptr_ty.elemType(), operand); - return self.finishAir(inst, result, .{ un_op, .none, .none }); } @@ -4773,69 +5621,6 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ .none, .none, .none }); } -fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u32 { - const abi_size = @intCast(u32, ty.abiSize(self.target.*)); - switch (condition) { - .none => unreachable, - .undef => unreachable, - .dead, .unreach => unreachable, - .eflags => unreachable, - .register => |cond_reg| { - try self.spillEflagsIfOccupied(); - - const cond_reg_lock = self.register_manager.lockReg(cond_reg); - defer if (cond_reg_lock) |lock| self.register_manager.unlockReg(lock); - - switch (case) { - .none => unreachable, - .undef => unreachable, - .dead, .unreach => unreachable, - .immediate => |imm| try self.asmRegisterImmediate( - .xor, - registerAlias(cond_reg, abi_size), - Immediate.u(imm), - ), - .register => |reg| try self.asmRegisterRegister( - .xor, - registerAlias(cond_reg, abi_size), - registerAlias(reg, abi_size), - ), - .stack_offset => { - if (abi_size <= 8) { - const reg = try self.copyToTmpRegister(ty, case); - return self.genCondSwitchMir(ty, condition, .{ .register = reg }); - } - - return self.fail("TODO implement switch mir when case is stack offset with abi larger than 8 bytes", .{}); - }, - else => { - return self.fail("TODO implement switch mir when case is {}", .{case}); - }, - } - - const aliased_reg = registerAlias(cond_reg, abi_size); - try self.asmRegisterRegister(.@"test", aliased_reg, aliased_reg); - return self.asmJccReloc(undefined, .ne); - }, - .stack_offset => { - try self.spillEflagsIfOccupied(); - - if (abi_size <= 8) { - const reg = try self.copyToTmpRegister(ty, condition); - const reg_lock = self.register_manager.lockRegAssumeUnused(reg); - defer self.register_manager.unlockReg(reg_lock); - return self.genCondSwitchMir(ty, .{ .register = reg }, case); - } - - return self.fail("TODO implement switch mir when condition is stack offset with abi larger than 8 bytes", .{}); - }, - else => { - return self.fail("TODO implemenent switch mir when condition is {}", .{condition}); - }, - } - return 0; // TODO -} - fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { const pl_op = self.air.instructions.items(.data)[inst].pl_op; const condition = try self.resolveInst(pl_op.operand); @@ -4880,8 +5665,10 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { defer self.gpa.free(relocs); for (items, relocs) |item, *reloc| { + try self.spillEflagsIfOccupied(); const item_mcv = try self.resolveInst(item); - reloc.* = try self.genCondSwitchMir(condition_ty, condition, item_mcv); + try self.genBinOpMir(.cmp, condition_ty, condition, item_mcv); + reloc.* = try self.asmJccReloc(undefined, .ne); } // Capture the state of register and stack allocation state so that we can revert to it. @@ -4976,14 +5763,8 @@ fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Bran if (target_value == .dead) continue; // The instruction is only overridden in the else branch. - var i: usize = self.branch_stack.items.len - 1; - while (true) { - i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead? - if (self.branch_stack.items[i].inst_table.get(target_key)) |mcv| { - assert(mcv != .dead); - break :blk mcv; - } - } + // If integer overflows occurs, the question is: why wasn't the instruction marked dead? + break :blk self.getResolvedInstValue(target_key).?; }; log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv }); // TODO make sure the destination stack offset / register does not already have something @@ -5000,16 +5781,7 @@ fn canonicaliseBranches(self: *Self, parent_branch: *Branch, canon_branch: *Bran log.debug("canon_value = {}", .{canon_value}); if (canon_value == .dead) continue; - const parent_mcv = blk: { - var i: usize = self.branch_stack.items.len - 1; - while (true) { - i -= 1; - if (self.branch_stack.items[i].inst_table.get(canon_key)) |mcv| { - assert(mcv != .dead); - break :blk mcv; - } - } - }; + const parent_mcv = self.getResolvedInstValue(canon_key).?; log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value }); // TODO make sure the destination stack offset / register does not already have something // going on there. @@ -5237,6 +6009,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .dead => unreachable, .unreach, .none => return, .undef => { + if (!self.wantSafety()) return; // The already existing value will do just fine. if (abi_size <= 8) { const reg = try self.copyToTmpRegister(ty, mcv); return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); @@ -5344,8 +6117,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl .dead => unreachable, .unreach, .none => return, // Nothing to do. .undef => { - if (!self.wantSafety()) - return; // The already existing value will do just fine. + if (!self.wantSafety()) return; // The already existing value will do just fine. // TODO Upgrade this to a memset call when we have that available. switch (abi_size) { 1, 2, 4 => { @@ -5567,19 +6339,14 @@ fn genInlineMemcpy( null; defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock); - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp); - const dst_addr_reg = regs[0]; - const src_addr_reg = regs[1]; - const index_reg = regs[2].to64(); - const count_reg = regs[3].to64(); - const tmp_reg = regs[4].to8(); + try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); switch (dst_ptr) { .memory, .linker_load => { - try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr); + try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr); }, .ptr_stack_offset, .stack_offset => |off| { - try self.asmRegisterMemory(.lea, dst_addr_reg.to64(), Memory.sib(.qword, .{ + try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{ .base = opts.dest_stack_base orelse .rbp, .disp = -off, })); @@ -5587,7 +6354,7 @@ fn genInlineMemcpy( .register => |reg| { try self.asmRegisterRegister( .mov, - registerAlias(dst_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))), + registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))), reg, ); }, @@ -5598,10 +6365,10 @@ fn genInlineMemcpy( switch (src_ptr) { .memory, .linker_load => { - try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr); + try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr); }, .ptr_stack_offset, .stack_offset => |off| { - try self.asmRegisterMemory(.lea, src_addr_reg.to64(), Memory.sib(.qword, .{ + try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{ .base = opts.source_stack_base orelse .rbp, .disp = -off, })); @@ -5609,7 +6376,7 @@ fn genInlineMemcpy( .register => |reg| { try self.asmRegisterRegister( .mov, - registerAlias(src_addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))), + registerAlias(.rsi, @intCast(u32, @divExact(reg.bitSize(), 8))), reg, ); }, @@ -5618,37 +6385,12 @@ fn genInlineMemcpy( }, } - try self.genSetReg(Type.usize, count_reg, len); - try self.asmRegisterImmediate(.mov, index_reg, Immediate.u(0)); - const loop_start = try self.addInst(.{ - .tag = .cmp, - .ops = .ri_u, - .data = .{ .ri = .{ - .r1 = count_reg, - .imm = 0, - } }, + try self.genSetReg(Type.usize, .rcx, len); + _ = try self.addInst(.{ + .tag = .movs, + .ops = .string, + .data = .{ .string = .{ .repeat = .rep, .width = .b } }, }); - const loop_reloc = try self.asmJccReloc(undefined, .e); - try self.asmRegisterMemory(.mov, tmp_reg.to8(), Memory.sib(.byte, .{ - .base = src_addr_reg, - .scale_index = .{ - .scale = 1, - .index = index_reg, - }, - .disp = 0, - })); - try self.asmMemoryRegister(.mov, Memory.sib(.byte, .{ - .base = dst_addr_reg, - .scale_index = .{ - .scale = 1, - .index = index_reg, - }, - .disp = 0, - }), tmp_reg.to8()); - try self.asmRegisterImmediate(.add, index_reg, Immediate.u(1)); - try self.asmRegisterImmediate(.sub, count_reg, Immediate.u(1)); - _ = try self.asmJmpReloc(loop_start); - try self.performReloc(loop_reloc); } fn genInlineMemset( @@ -5658,28 +6400,20 @@ fn genInlineMemset( len: MCValue, opts: InlineMemcpyOpts, ) InnerError!void { - const ssbase_lock: ?RegisterLock = if (opts.source_stack_base) |reg| - self.register_manager.lockReg(reg) - else - null; - defer if (ssbase_lock) |reg| self.register_manager.unlockReg(reg); - const dsbase_lock: ?RegisterLock = if (opts.dest_stack_base) |reg| self.register_manager.lockReg(reg) else null; defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock); - const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp); - const addr_reg = regs[0]; - const index_reg = regs[1].to64(); + try self.spillRegisters(&.{ .rdi, .al, .rcx }); switch (dst_ptr) { .memory, .linker_load => { - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr); + try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr); }, .ptr_stack_offset, .stack_offset => |off| { - try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{ + try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{ .base = opts.dest_stack_base orelse .rbp, .disp = -off, })); @@ -5687,48 +6421,22 @@ fn genInlineMemset( .register => |reg| { try self.asmRegisterRegister( .mov, - registerAlias(addr_reg, @intCast(u32, @divExact(reg.bitSize(), 8))), + registerAlias(.rdi, @intCast(u32, @divExact(reg.bitSize(), 8))), reg, ); }, else => { - return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr}); + return self.fail("TODO implement memset for setting stack when dest is {}", .{dst_ptr}); }, } - try self.genSetReg(Type.usize, index_reg, len); - try self.genBinOpMir(.sub, Type.usize, .{ .register = index_reg }, .{ .immediate = 1 }); - - const loop_start = try self.addInst(.{ - .tag = .cmp, - .ops = .ri_s, - .data = .{ .ri = .{ - .r1 = index_reg, - .imm = @bitCast(u32, @as(i32, -1)), - } }, + try self.genSetReg(Type.u8, .al, value); + try self.genSetReg(Type.usize, .rcx, len); + _ = try self.addInst(.{ + .tag = .stos, + .ops = .string, + .data = .{ .string = .{ .repeat = .rep, .width = .b } }, }); - const loop_reloc = try self.asmJccReloc(undefined, .e); - - switch (value) { - .immediate => |x| { - if (x > math.maxInt(i32)) { - return self.fail("TODO inline memset for value immediate larger than 32bits", .{}); - } - try self.asmMemoryImmediate(.mov, Memory.sib(.byte, .{ - .base = addr_reg, - .scale_index = .{ - .scale = 1, - .index = index_reg, - }, - .disp = 0, - }), Immediate.u(@intCast(u8, x))); - }, - else => return self.fail("TODO inline memset for value of type {}", .{value}), - } - - try self.asmRegisterImmediate(.sub, index_reg, Immediate.u(1)); - _ = try self.asmJmpReloc(loop_start); - try self.performReloc(loop_reloc); } fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void { @@ -5748,10 +6456,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, .unreach, .none => return, // Nothing to do. .undef => { - if (!self.wantSafety()) - return; // The already existing value will do just fine. + if (!self.wantSafety()) return; // The already existing value will do just fine. // Write the debug undefined value. - switch (registerAlias(reg, abi_size).bitSize()) { + switch (self.regBitSize(ty)) { 8 => return self.genSetReg(ty, reg, .{ .immediate = 0xaa }), 16 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaa }), 32 => return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaa }), @@ -5762,27 +6469,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .eflags => |cc| { return self.asmSetccRegister(reg.to8(), cc); }, - .immediate => |x| { - if (x == 0) { + .immediate => |imm| { + if (imm == 0) { // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit // register is the fastest way to zero a register. - return self.asmRegisterRegister(.xor, reg.to32(), reg.to32()); + try self.asmRegisterRegister(.xor, reg.to32(), reg.to32()); + } else if (abi_size > 4 and math.cast(u32, imm) != null) { + // 32-bit moves zero-extend to 64-bit. + try self.asmRegisterImmediate(.mov, reg.to32(), Immediate.u(imm)); + } else if (abi_size <= 4 and @bitCast(i64, imm) < 0) { + try self.asmRegisterImmediate( + .mov, + registerAlias(reg, abi_size), + Immediate.s(@intCast(i32, @bitCast(i64, imm))), + ); + } else { + try self.asmRegisterImmediate( + .mov, + registerAlias(reg, abi_size), + Immediate.u(imm), + ); } - if (ty.isSignedInt()) { - const signed_x = @bitCast(i64, x); - if (math.minInt(i32) <= signed_x and signed_x <= math.maxInt(i32)) { - return self.asmRegisterImmediate( - .mov, - registerAlias(reg, abi_size), - Immediate.s(@intCast(i32, signed_x)), - ); - } - } - return self.asmRegisterImmediate( - .mov, - registerAlias(reg, abi_size), - Immediate.u(x), - ); }, .register => |src_reg| { // If the registers are the same, nothing to do. @@ -5843,10 +6550,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .f64 => .qword, else => unreachable, }; - return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{ - .base = base_reg.to64(), - .disp = 0, - })); + return self.asmRegisterMemory( + tag, + reg.to128(), + Memory.sib(ptr_size, .{ .base = base_reg.to64() }), + ); } return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); @@ -5856,7 +6564,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void try self.asmRegisterMemory( .mov, registerAlias(reg, abi_size), - Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }), ); }, } @@ -5877,10 +6585,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .f64 => .qword, else => unreachable, }; - return self.asmRegisterMemory(tag, reg.to128(), Memory.sib(ptr_size, .{ - .base = base_reg.to64(), - .disp = 0, - })); + return self.asmRegisterMemory( + tag, + reg.to128(), + Memory.sib(ptr_size, .{ .base = base_reg.to64() }), + ); } return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); @@ -5916,7 +6625,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void try self.asmRegisterMemory( .mov, registerAlias(reg, abi_size), - Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64(), .disp = 0 }), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }), ); } } @@ -6088,26 +6797,178 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void { fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; - const extra = self.air.extraData(Air.Block, ty_pl.payload); - _ = extra; - return self.fail("TODO implement x86 airCmpxchg", .{}); - // return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); + const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + + const ptr_ty = self.air.typeOf(extra.ptr); + const ptr_mcv = try self.resolveInst(extra.ptr); + const val_ty = self.air.typeOf(extra.expected_value); + + const exp_mcv = try self.resolveInst(extra.expected_value); + try self.genSetReg(val_ty, .rax, exp_mcv); + const rax_lock = self.register_manager.lockRegAssumeUnused(.rax); + defer self.register_manager.unlockReg(rax_lock); + + const new_mcv = try self.resolveInst(extra.new_value); + const new_reg = try self.copyToTmpRegister(val_ty, new_mcv); + const new_lock = self.register_manager.lockRegAssumeUnused(new_reg); + defer self.register_manager.unlockReg(new_lock); + + const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*)); + const ptr_size = Memory.PtrSize.fromSize(val_abi_size); + const ptr_mem: Memory = switch (ptr_mcv) { + .register => |reg| Memory.sib(ptr_size, .{ .base = reg }), + .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }), + else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }), + }; + const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null; + defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); + + try self.spillEflagsIfOccupied(); + _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{ + .r1 = new_reg, + .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)), + } } }); + + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + self.eflags_inst = inst; + break :result .{ .register_overflow = .{ .reg = .rax, .eflags = .ne } }; + }; + return self.finishAir(inst, result, .{ extra.ptr, extra.expected_value, extra.new_value }); +} + +fn atomicOp( + self: *Self, + dst_reg: Register, + ptr_mcv: MCValue, + val_mcv: MCValue, + ptr_ty: Type, + val_ty: Type, + unused: bool, + op: ?std.builtin.AtomicRmwOp, + order: std.builtin.AtomicOrder, +) InnerError!void { + const dst_lock = self.register_manager.lockReg(dst_reg); + defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); + + const ptr_lock = switch (ptr_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const val_lock = switch (val_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (val_lock) |lock| self.register_manager.unlockReg(lock); + + const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*)); + const ptr_size = Memory.PtrSize.fromSize(val_abi_size); + const ptr_mem: Memory = switch (ptr_mcv) { + .register => |reg| Memory.sib(ptr_size, .{ .base = reg }), + .ptr_stack_offset => |off| Memory.sib(ptr_size, .{ .base = .rbp, .disp = -off }), + else => Memory.sib(ptr_size, .{ .base = try self.copyToTmpRegister(ptr_ty, ptr_mcv) }), + }; + const mem_lock = if (ptr_mem.base()) |reg| self.register_manager.lockReg(reg) else null; + defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genSetReg(val_ty, dst_reg, val_mcv); + + const need_loop = val_ty.isRuntimeFloat() or if (op) |rmw| switch (rmw) { + .Xchg, .Add, .Sub => false, + .And, .Or, .Xor => !unused, + .Nand, .Max, .Min => true, + } else false; + if (!need_loop) { + const tag: Mir.Inst.Tag = if (op) |rmw| switch (rmw) { + .Xchg => if (unused) .mov else .xchg, + .Add => if (unused) .add else .xadd, + .Sub => if (unused) .sub else .xadd, + .And => .@"and", + .Or => .@"or", + .Xor => .xor, + else => unreachable, + } else switch (order) { + .Unordered, .Monotonic, .Release, .AcqRel => .mov, + .Acquire => unreachable, + .SeqCst => .xchg, + }; + if (op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) { + try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg }); + } + _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) { + .mov, .xchg => .mr_sib, + .xadd, .add, .sub, .@"and", .@"or", .xor => .lock_mr_sib, + else => unreachable, + }, .data = .{ .rx = .{ + .r1 = registerAlias(dst_reg, val_abi_size), + .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)), + } } }); + return; + } + + return self.fail("TODO implement x86 atomic loop", .{}); } fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void { - _ = inst; - return self.fail("TODO implement x86 airAtomicRaw", .{}); + const pl_op = self.air.instructions.items(.data)[inst].pl_op; + const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; + + const dst_reg = try self.register_manager.allocReg(inst, gp); + + const ptr_ty = self.air.typeOf(pl_op.operand); + const ptr_mcv = try self.resolveInst(pl_op.operand); + + const val_ty = self.air.typeOf(extra.operand); + const val_mcv = try self.resolveInst(extra.operand); + + const unused = self.liveness.isUnused(inst); + try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering()); + const result: MCValue = if (unused) .dead else .{ .register = dst_reg }; + return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none }); } fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void { - _ = inst; - return self.fail("TODO implement airAtomicLoad for {}", .{self.target.cpu.arch}); + const atomic_load = self.air.instructions.items(.data)[inst].atomic_load; + + const result: MCValue = result: { + if (self.liveness.isUnused(inst)) break :result .dead; + + const ptr_ty = self.air.typeOf(atomic_load.ptr); + const ptr_mcv = try self.resolveInst(atomic_load.ptr); + const ptr_lock = switch (ptr_mcv) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = + if (self.reuseOperand(inst, atomic_load.ptr, 0, ptr_mcv)) + ptr_mcv + else + try self.allocRegOrMem(inst, true); + + try self.load(dst_mcv, ptr_mcv, ptr_ty); + break :result dst_mcv; + }; + return self.finishAir(inst, result, .{ atomic_load.ptr, .none, .none }); } fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void { - _ = inst; - _ = order; - return self.fail("TODO implement airAtomicStore for {}", .{self.target.cpu.arch}); + const bin_op = self.air.instructions.items(.data)[inst].bin_op; + + const dst_reg = try self.register_manager.allocReg(null, gp); + + const ptr_ty = self.air.typeOf(bin_op.lhs); + const ptr_mcv = try self.resolveInst(bin_op.lhs); + + const val_ty = self.air.typeOf(bin_op.rhs); + const val_mcv = try self.resolveInst(bin_op.rhs); + + try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order); + return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); } fn airMemset(self: *Self, inst: Air.Inst.Index) !void { @@ -6137,7 +6998,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void { try self.genInlineMemset(dst_ptr, src_val, len, .{}); - return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none }); + return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs }); } fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { @@ -6172,10 +7033,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { .linker_load, .memory => { const reg = try self.register_manager.allocReg(null, gp); try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); - try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ - .base = reg, - .disp = 0, - })); + try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg })); break :blk MCValue{ .register = reg }; }, else => break :blk src_ptr, @@ -6189,7 +7047,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { try self.genInlineMemcpy(dst_ptr, src, len, .{}); - return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none }); + return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs }); } fn airTagName(self: *Self, inst: Air.Inst.Index) !void { @@ -6629,7 +7487,10 @@ fn registerAlias(reg: Register, size_bytes: u32) Register { /// Truncates the value in the register in place. /// Clobbers any remaining bits. fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { - const int_info = ty.intInfo(self.target.*); + const int_info = if (ty.isAbiInt()) ty.intInfo(self.target.*) else std.builtin.Type.Int{ + .signedness = .unsigned, + .bits = @intCast(u16, ty.bitSize(self.target.*)), + }; const max_reg_bit_width = Register.rax.bitSize(); switch (int_info.signedness) { .signed => { @@ -6650,6 +7511,27 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { } } +fn regBitSize(self: *Self, ty: Type) u64 { + return switch (ty.zigTypeTag()) { + else => switch (ty.abiSize(self.target.*)) { + 1 => 8, + 2 => 16, + 3...4 => 32, + 5...8 => 64, + else => unreachable, + }, + .Float => switch (ty.abiSize(self.target.*)) { + 1...16 => 128, + 17...32 => 256, + else => unreachable, + }, + }; +} + +fn regExtraBits(self: *Self, ty: Type) u64 { + return self.regBitSize(ty) - ty.bitSize(self.target.*); +} + fn intrinsicsAllowed(target: Target, ty: Type) bool { return switch (ty.tag()) { .f32, diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 32699d35cb..cd8389aa49 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -73,6 +73,13 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .adc, .add, .@"and", + .bsf, + .bsr, + .bswap, + .bt, + .btc, + .btr, + .bts, .call, .cbw, .cwde, @@ -81,6 +88,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .cdq, .cqo, .cmp, + .cmpxchg, .div, .fisttp, .fld, @@ -89,35 +97,71 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .int3, .jmp, .lea, + .lfence, + .lzcnt, + .mfence, .mov, + .movbe, .movzx, .mul, + .neg, .nop, + .not, .@"or", .pop, + .popcnt, .push, + .rcl, + .rcr, .ret, + .rol, + .ror, .sal, .sar, .sbb, + .sfence, .shl, .shr, .sub, .syscall, .@"test", + .tzcnt, .ud2, + .xadd, + .xchg, .xor, .addss, .cmpss, + .divss, + .maxss, + .minss, .movss, + .mulss, + .roundss, + .subss, .ucomiss, .addsd, .cmpsd, + .divsd, + .maxsd, + .minsd, .movsd, + .mulsd, + .roundsd, + .subsd, .ucomisd, => try emit.mirEncodeGeneric(tag, inst), + .cmps, + .lods, + .movs, + .scas, + .stos, + => try emit.mirString(tag, inst), + + .cmpxchgb => try emit.mirCmpxchgBytes(inst), + .jmp_reloc => try emit.mirJmpReloc(inst), .call_extern => try emit.mirCallExtern(inst), @@ -171,18 +215,8 @@ fn fixupRelocs(emit: *Emit) InnerError!void { } } -fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: struct { - op1: Instruction.Operand = .none, - op2: Instruction.Operand = .none, - op3: Instruction.Operand = .none, - op4: Instruction.Operand = .none, -}) InnerError!void { - const inst = try Instruction.new(mnemonic, .{ - .op1 = ops.op1, - .op2 = ops.op2, - .op3 = ops.op3, - .op4 = ops.op4, - }); +fn encode(emit: *Emit, mnemonic: Instruction.Mnemonic, ops: Instruction.Init) InnerError!void { + const inst = try Instruction.new(mnemonic, ops); return inst.encode(emit.code.writer()); } @@ -194,6 +228,20 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE const ops = emit.mir.instructions.items(.ops)[inst]; const data = emit.mir.instructions.items(.data)[inst]; + const prefix: Instruction.Prefix = switch (ops) { + .lock_m_sib, + .lock_m_rip, + .lock_mi_u_sib, + .lock_mi_u_rip, + .lock_mi_s_sib, + .lock_mi_s_rip, + .lock_mr_sib, + .lock_mr_rip, + .lock_moffs_rax, + => .lock, + else => .none, + }; + var op1: Instruction.Operand = .none; var op2: Instruction.Operand = .none; var op3: Instruction.Operand = .none; @@ -232,35 +280,35 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE op2 = .{ .reg = data.rri.r2 }; op3 = .{ .imm = imm }; }, - .m_sib => { + .m_sib, .lock_m_sib => { const msib = emit.mir.extraData(Mir.MemorySib, data.payload).data; op1 = .{ .mem = Mir.MemorySib.decode(msib) }; }, - .m_rip => { + .m_rip, .lock_m_rip => { const mrip = emit.mir.extraData(Mir.MemoryRip, data.payload).data; op1 = .{ .mem = Mir.MemoryRip.decode(mrip) }; }, - .mi_s_sib, .mi_u_sib => { + .mi_s_sib, .mi_u_sib, .lock_mi_s_sib, .lock_mi_u_sib => { const msib = emit.mir.extraData(Mir.MemorySib, data.xi.payload).data; const imm = switch (ops) { - .mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)), - .mi_u_sib => Immediate.u(data.xi.imm), + .mi_s_sib, .lock_mi_s_sib => Immediate.s(@bitCast(i32, data.xi.imm)), + .mi_u_sib, .lock_mi_u_sib => Immediate.u(data.xi.imm), else => unreachable, }; op1 = .{ .mem = Mir.MemorySib.decode(msib) }; op2 = .{ .imm = imm }; }, - .mi_u_rip, .mi_s_rip => { + .mi_u_rip, .mi_s_rip, .lock_mi_u_rip, .lock_mi_s_rip => { const mrip = emit.mir.extraData(Mir.MemoryRip, data.xi.payload).data; const imm = switch (ops) { - .mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)), - .mi_u_rip => Immediate.u(data.xi.imm), + .mi_s_rip, .lock_mi_s_rip => Immediate.s(@bitCast(i32, data.xi.imm)), + .mi_u_rip, .lock_mi_u_rip => Immediate.u(data.xi.imm), else => unreachable, }; op1 = .{ .mem = Mir.MemoryRip.decode(mrip) }; op2 = .{ .imm = imm }; }, - .rm_sib, .mr_sib => { + .rm_sib, .mr_sib, .lock_mr_sib => { const msib = emit.mir.extraData(Mir.MemorySib, data.rx.payload).data; const op_r = .{ .reg = data.rx.r1 }; const op_m = .{ .mem = Mir.MemorySib.decode(msib) }; @@ -269,23 +317,23 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE op1 = op_r; op2 = op_m; }, - .mr_sib => { + .mr_sib, .lock_mr_sib => { op1 = op_m; op2 = op_r; }, else => unreachable, } }, - .rm_rip, .mr_rip => { + .rm_rip, .mr_rip, .lock_mr_rip => { const mrip = emit.mir.extraData(Mir.MemoryRip, data.rx.payload).data; const op_r = .{ .reg = data.rx.r1 }; const op_m = .{ .mem = Mir.MemoryRip.decode(mrip) }; switch (ops) { - .rm_sib => { + .rm_rip => { op1 = op_r; op2 = op_m; }, - .mr_sib => { + .mr_rip, .lock_mr_rip => { op1 = op_m; op2 = op_r; }, @@ -299,6 +347,7 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE } return emit.encode(mnemonic, .{ + .prefix = prefix, .op1 = op1, .op2 = op2, .op3 = op3, @@ -306,6 +355,61 @@ fn mirEncodeGeneric(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE }); } +fn mirString(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { + const ops = emit.mir.instructions.items(.ops)[inst]; + switch (ops) { + .string => { + const data = emit.mir.instructions.items(.data)[inst].string; + const mnemonic = switch (tag) { + inline .cmps, .lods, .movs, .scas, .stos => |comptime_tag| switch (data.width) { + inline else => |comptime_width| @field( + Instruction.Mnemonic, + @tagName(comptime_tag) ++ @tagName(comptime_width), + ), + }, + else => unreachable, + }; + return emit.encode(mnemonic, .{ .prefix = switch (data.repeat) { + inline else => |comptime_repeat| @field(Instruction.Prefix, @tagName(comptime_repeat)), + } }); + }, + else => unreachable, + } +} + +fn mirCmpxchgBytes(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { + const ops = emit.mir.instructions.items(.ops)[inst]; + const data = emit.mir.instructions.items(.data)[inst]; + + var op1: Instruction.Operand = .none; + switch (ops) { + .m_sib, .lock_m_sib => { + const sib = emit.mir.extraData(Mir.MemorySib, data.payload).data; + op1 = .{ .mem = Mir.MemorySib.decode(sib) }; + }, + .m_rip, .lock_m_rip => { + const rip = emit.mir.extraData(Mir.MemoryRip, data.payload).data; + op1 = .{ .mem = Mir.MemoryRip.decode(rip) }; + }, + else => unreachable, + } + + const mnemonic: Instruction.Mnemonic = switch (op1.mem.bitSize()) { + 64 => .cmpxchg8b, + 128 => .cmpxchg16b, + else => unreachable, + }; + + return emit.encode(mnemonic, .{ + .prefix = switch (ops) { + .m_sib, .m_rip => .none, + .lock_m_sib, .lock_m_rip => .lock, + else => unreachable, + }, + .op1 = op1, + }); +} + fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst]; const payload = emit.mir.instructions.items(.data)[inst].payload; @@ -319,8 +423,13 @@ fn mirMovMoffs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { .op2 = .{ .mem = Memory.moffs(seg, offset) }, }); }, - .moffs_rax => { + .moffs_rax, .lock_moffs_rax => { try emit.encode(.mov, .{ + .prefix = switch (ops) { + .moffs_rax => .none, + .lock_moffs_rax => .lock, + else => unreachable, + }, .op1 = .{ .mem = Memory.moffs(seg, offset) }, .op2 = .{ .reg = .rax }, }); @@ -365,37 +474,70 @@ fn mirMovsx(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } fn mnemonicFromConditionCode(comptime basename: []const u8, cc: bits.Condition) Instruction.Mnemonic { - inline for (@typeInfo(bits.Condition).Enum.fields) |field| { - if (mem.eql(u8, field.name, @tagName(cc))) - return @field(Instruction.Mnemonic, basename ++ field.name); - } else unreachable; + return switch (cc) { + inline else => |comptime_cc| @field(Instruction.Mnemonic, basename ++ @tagName(comptime_cc)), + }; } fn mirCmovcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst]; switch (ops) { - .rr_c => { - const data = emit.mir.instructions.items(.data)[inst].rr_c; + .rr_cc => { + const data = emit.mir.instructions.items(.data)[inst].rr_cc; const mnemonic = mnemonicFromConditionCode("cmov", data.cc); return emit.encode(mnemonic, .{ .op1 = .{ .reg = data.r1 }, .op2 = .{ .reg = data.r2 }, }); }, - else => unreachable, // TODO + .rm_sib_cc => { + const data = emit.mir.instructions.items(.data)[inst].rx_cc; + const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data; + const mnemonic = mnemonicFromConditionCode("cmov", data.cc); + return emit.encode(mnemonic, .{ + .op1 = .{ .reg = data.r1 }, + .op2 = .{ .mem = Mir.MemorySib.decode(extra) }, + }); + }, + .rm_rip_cc => { + const data = emit.mir.instructions.items(.data)[inst].rx_cc; + const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data; + const mnemonic = mnemonicFromConditionCode("cmov", data.cc); + return emit.encode(mnemonic, .{ + .op1 = .{ .reg = data.r1 }, + .op2 = .{ .mem = Mir.MemoryRip.decode(extra) }, + }); + }, + else => unreachable, } } fn mirSetcc(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst]; switch (ops) { - .r_c => { - const data = emit.mir.instructions.items(.data)[inst].r_c; + .r_cc => { + const data = emit.mir.instructions.items(.data)[inst].r_cc; const mnemonic = mnemonicFromConditionCode("set", data.cc); return emit.encode(mnemonic, .{ .op1 = .{ .reg = data.r1 }, }); }, + .m_sib_cc => { + const data = emit.mir.instructions.items(.data)[inst].x_cc; + const extra = emit.mir.extraData(Mir.MemorySib, data.payload).data; + const mnemonic = mnemonicFromConditionCode("set", data.cc); + return emit.encode(mnemonic, .{ + .op1 = .{ .mem = Mir.MemorySib.decode(extra) }, + }); + }, + .m_rip_cc => { + const data = emit.mir.instructions.items(.data)[inst].x_cc; + const extra = emit.mir.extraData(Mir.MemoryRip, data.payload).data; + const mnemonic = mnemonicFromConditionCode("set", data.cc); + return emit.encode(mnemonic, .{ + .op1 = .{ .mem = Mir.MemoryRip.decode(extra) }, + }); + }, else => unreachable, // TODO } } diff --git a/src/arch/x86_64/Encoding.zig b/src/arch/x86_64/Encoding.zig index a51f954aed..891fc4e9a1 100644 --- a/src/arch/x86_64/Encoding.zig +++ b/src/arch/x86_64/Encoding.zig @@ -19,17 +19,12 @@ op1: Op, op2: Op, op3: Op, op4: Op, -opc_len: u2, -opc: [3]u8, +opc_len: u3, +opc: [7]u8, modrm_ext: u3, mode: Mode, -pub fn findByMnemonic(mnemonic: Mnemonic, args: struct { - op1: Instruction.Operand, - op2: Instruction.Operand, - op3: Instruction.Operand, - op4: Instruction.Operand, -}) !?Encoding { +pub fn findByMnemonic(mnemonic: Mnemonic, args: Instruction.Init) !?Encoding { const input_op1 = Op.fromOperand(args.op1); const input_op2 = Op.fromOperand(args.op2); const input_op3 = Op.fromOperand(args.op3); @@ -69,18 +64,19 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct { var candidates: [10]Encoding = undefined; var count: usize = 0; for (table) |entry| { - const enc = Encoding{ + var enc = Encoding{ .mnemonic = entry[0], .op_en = entry[1], .op1 = entry[2], .op2 = entry[3], .op3 = entry[4], .op4 = entry[5], - .opc_len = entry[6], - .opc = .{ entry[7], entry[8], entry[9] }, - .modrm_ext = entry[10], - .mode = entry[11], + .opc_len = @intCast(u3, entry[6].len), + .opc = undefined, + .modrm_ext = entry[7], + .mode = entry[8], }; + std.mem.copy(u8, &enc.opc, entry[6]); if (enc.mnemonic == mnemonic and input_op1.isSubset(enc.op1, enc.mode) and input_op2.isSubset(enc.op2, enc.mode) and @@ -108,17 +104,13 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct { if (count == 1) return candidates[0]; const EncodingLength = struct { - fn estimate(encoding: Encoding, params: struct { - op1: Instruction.Operand, - op2: Instruction.Operand, - op3: Instruction.Operand, - op4: Instruction.Operand, - }) usize { + fn estimate(encoding: Encoding, params: Instruction.Init) usize { var inst = Instruction{ .op1 = params.op1, .op2 = params.op2, .op3 = params.op3, .op4 = params.op4, + .prefix = params.prefix, .encoding = encoding, }; var cwriter = std.io.countingWriter(std.io.null_writer); @@ -139,12 +131,7 @@ pub fn findByMnemonic(mnemonic: Mnemonic, args: struct { else => {}, } - const len = EncodingLength.estimate(candidate, .{ - .op1 = args.op1, - .op2 = args.op2, - .op3 = args.op3, - .op4 = args.op4, - }); + const len = EncodingLength.estimate(candidate, args); const current = shortest_encoding orelse { shortest_encoding = .{ .index = i, .len = len }; continue; @@ -184,7 +171,7 @@ pub fn findByOpcode(opc: []const u8, prefixes: struct { if (match) { if (prefixes.rex.w) { switch (enc.mode) { - .fpu, .sse, .sse2, .none => {}, + .fpu, .sse, .sse2, .sse4_1, .none => {}, .long, .rex => return enc, } } else if (prefixes.rex.present and !prefixes.rex.isSet()) { @@ -227,7 +214,11 @@ pub fn modRmExt(encoding: Encoding) u3 { } pub fn operandBitSize(encoding: Encoding) u64 { - if (encoding.mode == .long) return 64; + switch (encoding.mode) { + .short => return 16, + .long => return 64, + else => {}, + } const bit_size: u64 = switch (encoding.op_en) { .np => switch (encoding.op1) { .o16 => 16, @@ -316,39 +307,63 @@ pub const Mnemonic = enum { // zig fmt: off // General-purpose adc, add, @"and", - call, cbw, cwde, cdqe, cwd, cdq, cqo, cmp, + bsf, bsr, bswap, bt, btc, btr, bts, + call, cbw, cdq, cdqe, cmova, cmovae, cmovb, cmovbe, cmovc, cmove, cmovg, cmovge, cmovl, cmovle, cmovna, cmovnae, cmovnb, cmovnbe, cmovnc, cmovne, cmovng, cmovnge, cmovnl, cmovnle, cmovno, cmovnp, cmovns, cmovnz, cmovo, cmovp, cmovpe, cmovpo, cmovs, cmovz, + cmp, + cmps, cmpsb, cmpsd, cmpsq, cmpsw, + cmpxchg, cmpxchg8b, cmpxchg16b, + cqo, cwd, cwde, div, fisttp, fld, idiv, imul, int3, ja, jae, jb, jbe, jc, jrcxz, je, jg, jge, jl, jle, jna, jnae, jnb, jnbe, jnc, jne, jng, jnge, jnl, jnle, jno, jnp, jns, jnz, jo, jp, jpe, jpo, js, jz, jmp, - lea, - mov, movsx, movsxd, movzx, mul, - nop, + lea, lfence, + lods, lodsb, lodsd, lodsq, lodsw, + lzcnt, + mfence, mov, movbe, + movs, movsb, movsd, movsq, movsw, + movsx, movsxd, movzx, mul, + neg, nop, not, @"or", - pop, push, - ret, - sal, sar, sbb, shl, shr, sub, syscall, + pop, popcnt, push, + rcl, rcr, ret, rol, ror, + sal, sar, sbb, + scas, scasb, scasd, scasq, scasw, + shl, shr, sub, syscall, seta, setae, setb, setbe, setc, sete, setg, setge, setl, setle, setna, setnae, setnb, setnbe, setnc, setne, setng, setnge, setnl, setnle, setno, setnp, setns, setnz, seto, setp, setpe, setpo, sets, setz, - @"test", + sfence, + stos, stosb, stosd, stosq, stosw, + @"test", tzcnt, ud2, - xor, + xadd, xchg, xor, // SSE addss, cmpss, + divss, + maxss, minss, movss, + mulss, + subss, ucomiss, // SSE2 addsd, - cmpsd, - movq, movsd, + //cmpsd, + divsd, + maxsd, minsd, + movq, //movsd, + mulsd, + subsd, ucomisd, + // SSE4.1 + roundss, + roundsd, // zig fmt: on }; @@ -374,7 +389,7 @@ pub const Op = enum { cl, r8, r16, r32, r64, rm8, rm16, rm32, rm64, - m8, m16, m32, m64, m80, + m8, m16, m32, m64, m80, m128, rel8, rel16, rel32, m, moffs, @@ -423,6 +438,7 @@ pub const Op = enum { 32 => .m32, 64 => .m64, 80 => .m80, + 128 => .m128, else => unreachable, }; }, @@ -460,7 +476,7 @@ pub const Op = enum { .imm32, .imm32s, .eax, .r32, .m32, .rm32, .rel32, .xmm_m32 => 32, .imm64, .rax, .r64, .m64, .rm64, .xmm_m64 => 64, .m80 => 80, - .xmm => 128, + .m128, .xmm => 128, }; } @@ -507,7 +523,7 @@ pub const Op = enum { // zig fmt: off return switch (op) { .rm8, .rm16, .rm32, .rm64, - .m8, .m16, .m32, .m64, .m80, + .m8, .m16, .m32, .m64, .m80, .m128, .m, .xmm_m32, .xmm_m64, => true, @@ -542,7 +558,7 @@ pub const Op = enum { else => { if (op.isRegister() and target.isRegister()) { switch (mode) { - .sse, .sse2 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(), + .sse, .sse2, .sse4_1 => return op.isFloatingPointRegister() and target.isFloatingPointRegister(), else => switch (target) { .cl, .al, .ax, .eax, .rax => return op == target, else => return op.bitSize() == target.bitSize(), @@ -579,9 +595,11 @@ pub const Op = enum { pub const Mode = enum { none, + short, fpu, rex, long, sse, sse2, + sse4_1, }; diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 3951108e3a..59c292c500 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -38,6 +38,20 @@ pub const Inst = struct { add, /// Logical and @"and", + /// Bit scan forward + bsf, + /// Bit scan reverse + bsr, + /// Byte swap + bswap, + /// Bit test + bt, + /// Bit test and complement + btc, + /// Bit test and reset + btr, + /// Bit test and set + bts, /// Call call, /// Convert byte to word @@ -54,6 +68,10 @@ pub const Inst = struct { cqo, /// Logical compare cmp, + /// Compare and exchange + cmpxchg, + /// Compare and exchange bytes + cmpxchgb, /// Unsigned division div, /// Store integer with truncation @@ -70,30 +88,54 @@ pub const Inst = struct { jmp, /// Load effective address lea, + /// Load fence + lfence, + /// Count the number of leading zero bits + lzcnt, + /// Memory fence + mfence, /// Move mov, + /// Move data after swapping bytes + movbe, /// Move with sign extension movsx, /// Move with zero extension movzx, /// Multiply mul, + /// Two's complement negation + neg, /// No-op nop, + /// One's complement negation + not, /// Logical or @"or", /// Pop pop, + /// Return the count of number of bits set to 1 + popcnt, /// Push push, + /// Rotate left through carry + rcl, + /// Rotate right through carry + rcr, /// Return ret, + /// Rotate left + rol, + /// Rotate right + ror, /// Arithmetic shift left sal, /// Arithmetic shift right sar, /// Integer subtraction with borrow sbb, + /// Store fence + sfence, /// Logical shift left shl, /// Logical shift right @@ -104,28 +146,69 @@ pub const Inst = struct { syscall, /// Test condition @"test", + /// Count the number of trailing zero bits + tzcnt, /// Undefined instruction ud2, + /// Exchange and add + xadd, + /// Exchange register/memory with register + xchg, /// Logical exclusive-or xor, - /// Add single precision floating point + /// Add single precision floating point values addss, /// Compare scalar single-precision floating-point values cmpss, + /// Divide scalar single-precision floating-point values + divss, + /// Return maximum single-precision floating-point value + maxss, + /// Return minimum single-precision floating-point value + minss, /// Move scalar single-precision floating-point value movss, + /// Multiply scalar single-precision floating-point values + mulss, + /// Round scalar single-precision floating-point values + roundss, + /// Subtract scalar single-precision floating-point values + subss, /// Unordered compare scalar single-precision floating-point values ucomiss, - /// Add double precision floating point + /// Add double precision floating point values addsd, /// Compare scalar double-precision floating-point values cmpsd, + /// Divide scalar double-precision floating-point values + divsd, + /// Return maximum double-precision floating-point value + maxsd, + /// Return minimum double-precision floating-point value + minsd, /// Move scalar double-precision floating-point value movsd, + /// Multiply scalar double-precision floating-point values + mulsd, + /// Round scalar double-precision floating-point values + roundsd, + /// Subtract scalar double-precision floating-point values + subsd, /// Unordered compare scalar double-precision floating-point values ucomisd, + /// Compare string operands + cmps, + /// Load string + lods, + /// Move data from string to string + movs, + /// Scan string + scas, + /// Store string + stos, + /// Conditional move cmovcc, /// Conditional jump @@ -185,11 +268,11 @@ pub const Inst = struct { /// Uses `rri` payload. rri_u, /// Register with condition code (CC). - /// Uses `r_c` payload. - r_c, + /// Uses `r_cc` payload. + r_cc, /// Register, register with condition code (CC). - /// Uses `rr_c` payload. - rr_c, + /// Uses `rr_cc` payload. + rr_cc, /// Register, immediate (sign-extended) operands. /// Uses `ri` payload. ri_s, @@ -214,12 +297,24 @@ pub const Inst = struct { /// Register, memory (RIP) operands. /// Uses `rx` payload. rm_rip, + /// Register, memory (SIB) operands with condition code (CC). + /// Uses `rx_cc` payload. + rm_sib_cc, + /// Register, memory (RIP) operands with condition code (CC). + /// Uses `rx_cc` payload. + rm_rip_cc, /// Single memory (SIB) operand. /// Uses `payload` with extra data of type `MemorySib`. m_sib, /// Single memory (RIP) operand. /// Uses `payload` with extra data of type `MemoryRip`. m_rip, + /// Single memory (SIB) operand with condition code (CC). + /// Uses `x_cc` with extra data of type `MemorySib`. + m_sib_cc, + /// Single memory (RIP) operand with condition code (CC). + /// Uses `x_cc` with extra data of type `MemoryRip`. + m_rip_cc, /// Memory (SIB), immediate (unsigned) operands. /// Uses `xi` payload with extra data of type `MemorySib`. mi_u_sib, @@ -244,16 +339,42 @@ pub const Inst = struct { /// Memory moffs, rax. /// Uses `payload` with extra data of type `MemoryMoffs`. moffs_rax, + /// Single memory (SIB) operand with lock prefix. + /// Uses `payload` with extra data of type `MemorySib`. + lock_m_sib, + /// Single memory (RIP) operand with lock prefix. + /// Uses `payload` with extra data of type `MemoryRip`. + lock_m_rip, + /// Memory (SIB), immediate (unsigned) operands with lock prefix. + /// Uses `xi` payload with extra data of type `MemorySib`. + lock_mi_u_sib, + /// Memory (RIP), immediate (unsigned) operands with lock prefix. + /// Uses `xi` payload with extra data of type `MemoryRip`. + lock_mi_u_rip, + /// Memory (SIB), immediate (sign-extend) operands with lock prefix. + /// Uses `xi` payload with extra data of type `MemorySib`. + lock_mi_s_sib, + /// Memory (RIP), immediate (sign-extend) operands with lock prefix. + /// Uses `xi` payload with extra data of type `MemoryRip`. + lock_mi_s_rip, + /// Memory (SIB), register operands with lock prefix. + /// Uses `rx` payload with extra data of type `MemorySib`. + lock_mr_sib, + /// Memory (RIP), register operands with lock prefix. + /// Uses `rx` payload with extra data of type `MemoryRip`. + lock_mr_rip, + /// Memory moffs, rax with lock prefix. + /// Uses `payload` with extra data of type `MemoryMoffs`. + lock_moffs_rax, /// References another Mir instruction directly. /// Uses `inst` payload. inst, /// References another Mir instruction directly with condition code (CC). /// Uses `inst_cc` payload. inst_cc, - /// Uses `payload` payload with data of type `MemoryConditionCode`. - m_cc, - /// Uses `rx` payload with extra data of type `MemoryConditionCode`. - rm_cc, + /// String repeat and width + /// Uses `string` payload. + string, /// Uses `reloc` payload. reloc, /// Linker relocation - GOT indirection. @@ -295,13 +416,18 @@ pub const Inst = struct { r2: Register, imm: u32, }, + /// Condition code (CC), followed by custom payload found in extra. + x_cc: struct { + payload: u32, + cc: bits.Condition, + }, /// Register with condition code (CC). - r_c: struct { + r_cc: struct { r1: Register, cc: bits.Condition, }, /// Register, register with condition code (CC). - rr_c: struct { + rr_cc: struct { r1: Register, r2: Register, cc: bits.Condition, @@ -316,11 +442,22 @@ pub const Inst = struct { r1: Register, payload: u32, }, + /// Register with condition code (CC), followed by custom payload found in extra. + rx_cc: struct { + r1: Register, + cc: bits.Condition, + payload: u32, + }, /// Custom payload followed by an immediate. xi: struct { payload: u32, imm: u32, }, + /// String instruction prefix and width. + string: struct { + repeat: bits.StringRepeat, + width: bits.StringWidth, + }, /// Relocation for the linker where: /// * `atom_index` is the index of the source /// * `sym_index` is the index of the target diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 043e589af4..c10bfe4039 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -6,6 +6,9 @@ const Allocator = std.mem.Allocator; const ArrayList = std.ArrayList; const DW = std.dwarf; +pub const StringRepeat = enum(u3) { none, rep, repe, repz, repne, repnz }; +pub const StringWidth = enum(u2) { b, w, d, q }; + /// EFLAGS condition codes pub const Condition = enum(u5) { /// above @@ -97,8 +100,7 @@ pub const Condition = enum(u5) { }; } - /// Returns the condition which is true iff the given condition is - /// false (if such a condition exists) + /// Returns the condition which is true iff the given condition is false pub fn negate(cond: Condition) Condition { return switch (cond) { .a => .na, @@ -127,8 +129,8 @@ pub const Condition = enum(u5) { .nz => .z, .o => .no, .p => .np, - .pe => unreachable, - .po => unreachable, + .pe => .po, + .po => .pe, .s => .ns, .z => .nz, }; @@ -470,10 +472,11 @@ pub const Memory = union(enum) { } pub fn sib(ptr_size: PtrSize, args: struct { - disp: i32, + disp: i32 = 0, base: ?Register = null, scale_index: ?ScaleIndex = null, }) Memory { + if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale)); return .{ .sib = .{ .base = args.base, .disp = args.disp, diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig index 7e29f95069..b3de7ec1bd 100644 --- a/src/arch/x86_64/encoder.zig +++ b/src/arch/x86_64/encoder.zig @@ -15,10 +15,21 @@ pub const Instruction = struct { op2: Operand = .none, op3: Operand = .none, op4: Operand = .none, + prefix: Prefix = .none, encoding: Encoding, pub const Mnemonic = Encoding.Mnemonic; + pub const Prefix = enum(u3) { + none, + lock, + rep, + repe, + repz, + repne, + repnz, + }; + pub const Operand = union(enum) { none, reg: Register, @@ -96,19 +107,18 @@ pub const Instruction = struct { } }; - pub fn new(mnemonic: Mnemonic, args: struct { + pub const Init = struct { + prefix: Prefix = .none, op1: Operand = .none, op2: Operand = .none, op3: Operand = .none, op4: Operand = .none, - }) !Instruction { - const encoding = (try Encoding.findByMnemonic(mnemonic, .{ - .op1 = args.op1, - .op2 = args.op2, - .op3 = args.op3, - .op4 = args.op4, - })) orelse { - log.debug("no encoding found for: {s} {s} {s} {s} {s}", .{ + }; + + pub fn new(mnemonic: Mnemonic, args: Init) !Instruction { + const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse { + log.debug("no encoding found for: {s} {s} {s} {s} {s} {s}", .{ + @tagName(args.prefix), @tagName(mnemonic), @tagName(Encoding.Op.fromOperand(args.op1)), @tagName(Encoding.Op.fromOperand(args.op2)), @@ -119,6 +129,7 @@ pub const Instruction = struct { }; log.debug("selected encoding: {}", .{encoding}); return .{ + .prefix = args.prefix, .op1 = args.op1, .op2 = args.op2, .op3 = args.op3, @@ -128,6 +139,7 @@ pub const Instruction = struct { } pub fn fmtPrint(inst: Instruction, writer: anytype) !void { + if (inst.prefix != .none) try writer.print("{s} ", .{@tagName(inst.prefix)}); try writer.print("{s}", .{@tagName(inst.encoding.mnemonic)}); const ops = [_]struct { Operand, Encoding.Op }{ .{ inst.op1, inst.encoding.op1 }, @@ -199,14 +211,12 @@ pub const Instruction = struct { fn encodeOpcode(inst: Instruction, encoder: anytype) !void { const opcode = inst.encoding.opcode(); + const first = @boolToInt(inst.encoding.mandatoryPrefix() != null); + const final = opcode.len - 1; + for (opcode[first..final]) |byte| try encoder.opcode_1byte(byte); switch (inst.encoding.op_en) { - .o, .oi => try encoder.opcode_withReg(opcode[0], inst.op1.reg.lowEnc()), - else => { - const index: usize = if (inst.encoding.mandatoryPrefix()) |_| 1 else 0; - for (opcode[index..]) |byte| { - try encoder.opcode_1byte(byte); - } - }, + .o, .oi => try encoder.opcode_withReg(opcode[final], inst.op1.reg.lowEnc()), + else => try encoder.opcode_1byte(opcode[final]), } } @@ -215,6 +225,14 @@ pub const Instruction = struct { const op_en = enc.op_en; var legacy = LegacyPrefixes{}; + + switch (inst.prefix) { + .none => {}, + .lock => legacy.prefix_f0 = true, + .repne, .repnz => legacy.prefix_f2 = true, + .rep, .repe, .repz => legacy.prefix_f3 = true, + } + if (enc.mode == .none) { const bit_size = enc.operandBitSize(); if (bit_size == 16) { @@ -811,15 +829,11 @@ const TestEncode = struct { buffer: [32]u8 = undefined, index: usize = 0, - fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: struct { - op1: Instruction.Operand = .none, - op2: Instruction.Operand = .none, - op3: Instruction.Operand = .none, - op4: Instruction.Operand = .none, - }) !void { + fn encode(enc: *TestEncode, mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void { var stream = std.io.fixedBufferStream(&enc.buffer); var count_writer = std.io.countingWriter(stream.writer()); const inst = try Instruction.new(mnemonic, .{ + .prefix = args.prefix, .op1 = args.op1, .op2 = args.op2, .op3 = args.op3, @@ -880,10 +894,10 @@ test "lower MI encoding" { try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); - try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.byte, .{ - .base = .r12, - .disp = 0, - }) }, .op2 = .{ .imm = Immediate.u(0x10) } }); + try enc.encode(.mov, .{ + .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) }, + .op2 = .{ .imm = Immediate.u(0x10) }, + }); try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10"); try enc.encode(.mov, .{ .op1 = .{ .reg = .r12 }, .op2 = .{ .imm = Immediate.u(0x1000) } }); @@ -895,10 +909,10 @@ test "lower MI encoding" { try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .imm = Immediate.u(0x10) } }); try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10"); - try enc.encode(.mov, .{ .op1 = .{ .mem = Memory.sib(.dword, .{ - .base = .r11, - .disp = 0, - }) }, .op2 = .{ .imm = Immediate.u(0x10) } }); + try enc.encode(.mov, .{ + .op1 = .{ .mem = Memory.sib(.dword, .{ .base = .r11 }) }, + .op2 = .{ .imm = Immediate.u(0x10) }, + }); try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10"); try enc.encode(.mov, .{ @@ -1014,10 +1028,10 @@ test "lower MI encoding" { test "lower RM encoding" { var enc = TestEncode{}; - try enc.encode(.mov, .{ .op1 = .{ .reg = .rax }, .op2 = .{ .mem = Memory.sib(.qword, .{ - .base = .r11, - .disp = 0, - }) } }); + try enc.encode(.mov, .{ + .op1 = .{ .reg = .rax }, + .op2 = .{ .mem = Memory.sib(.qword, .{ .base = .r11 }) }, + }); try expectEqualHexStrings("\x49\x8b\x03", enc.code(), "mov rax, QWORD PTR [r11]"); try enc.encode(.mov, .{ .op1 = .{ .reg = .rbx }, .op2 = .{ .mem = Memory.sib(.qword, .{ @@ -1100,20 +1114,16 @@ test "lower RM encoding" { try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .reg = .bl } }); try expectEqualHexStrings("\x66\x0F\xBE\xC3", enc.code(), "movsx ax, bl"); - try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.word, .{ - .base = .rbp, - .disp = 0, - }) } }); + try enc.encode(.movsx, .{ + .op1 = .{ .reg = .eax }, + .op2 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) }, + }); try expectEqualHexStrings("\x0F\xBF\x45\x00", enc.code(), "movsx eax, BYTE PTR [rbp]"); - try enc.encode(.movsx, .{ .op1 = .{ .reg = .eax }, .op2 = .{ .mem = Memory.sib(.byte, .{ - .base = null, - .scale_index = .{ - .index = .rax, - .scale = 2, - }, - .disp = 0, - }) } }); + try enc.encode(.movsx, .{ + .op1 = .{ .reg = .eax }, + .op2 = .{ .mem = Memory.sib(.byte, .{ .scale_index = .{ .index = .rax, .scale = 2 } }) }, + }); try expectEqualHexStrings("\x0F\xBE\x04\x45\x00\x00\x00\x00", enc.code(), "movsx eax, BYTE PTR [rax * 2]"); try enc.encode(.movsx, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } }); @@ -1140,14 +1150,13 @@ test "lower RM encoding" { try enc.encode(.lea, .{ .op1 = .{ .reg = .ax }, .op2 = .{ .mem = Memory.rip(.byte, 0x10) } }); try expectEqualHexStrings("\x66\x8D\x05\x10\x00\x00\x00", enc.code(), "lea ax, BYTE PTR [rip + 0x10]"); - try enc.encode(.lea, .{ .op1 = .{ .reg = .rsi }, .op2 = .{ .mem = Memory.sib(.qword, .{ - .base = .rbp, - .scale_index = .{ - .scale = 1, - .index = .rcx, - }, - .disp = 0, - }) } }); + try enc.encode(.lea, .{ + .op1 = .{ .reg = .rsi }, + .op2 = .{ .mem = Memory.sib(.qword, .{ + .base = .rbp, + .scale_index = .{ .scale = 1, .index = .rcx }, + }) }, + }); try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", enc.code(), "lea rsi, QWORD PTR [rbp + rcx*1 + 0]"); try enc.encode(.add, .{ .op1 = .{ .reg = .r11 }, .op2 = .{ .mem = Memory.sib(.qword, .{ @@ -1303,51 +1312,35 @@ test "lower M encoding" { try enc.encode(.call, .{ .op1 = .{ .reg = .r12 } }); try expectEqualHexStrings("\x41\xFF\xD4", enc.code(), "call r12"); - try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ - .base = .r12, - .disp = 0, - }) } }); + try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .r12 }) } }); try expectEqualHexStrings("\x41\xFF\x14\x24", enc.code(), "call QWORD PTR [r12]"); - try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ - .base = null, - .scale_index = .{ - .index = .r11, - .scale = 2, - }, - .disp = 0, - }) } }); + try enc.encode(.call, .{ + .op1 = .{ .mem = Memory.sib(.qword, .{ + .base = null, + .scale_index = .{ .index = .r11, .scale = 2 }, + }) }, + }); try expectEqualHexStrings("\x42\xFF\x14\x5D\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r11 * 2]"); - try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ - .base = null, - .scale_index = .{ - .index = .r12, - .scale = 2, - }, - .disp = 0, - }) } }); + try enc.encode(.call, .{ + .op1 = .{ .mem = Memory.sib(.qword, .{ + .base = null, + .scale_index = .{ .index = .r12, .scale = 2 }, + }) }, + }); try expectEqualHexStrings("\x42\xFF\x14\x65\x00\x00\x00\x00", enc.code(), "call QWORD PTR [r12 * 2]"); - try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ - .base = .gs, - .disp = 0, - }) } }); + try enc.encode(.call, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .gs }) } }); try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0"); try enc.encode(.call, .{ .op1 = .{ .imm = Immediate.s(0) } }); try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0"); - try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ - .base = .rbp, - .disp = 0, - }) } }); + try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.qword, .{ .base = .rbp }) } }); try expectEqualHexStrings("\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]"); - try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{ - .base = .rbp, - .disp = 0, - }) } }); + try enc.encode(.push, .{ .op1 = .{ .mem = Memory.sib(.word, .{ .base = .rbp }) } }); try expectEqualHexStrings("\x66\xFF\x75\x00", enc.code(), "push QWORD PTR [rbp]"); try enc.encode(.pop, .{ .op1 = .{ .mem = Memory.rip(.qword, 0) } }); @@ -1447,18 +1440,8 @@ test "lower NP encoding" { try expectEqualHexStrings("\x0f\x05", enc.code(), "syscall"); } -fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: struct { - op1: Instruction.Operand = .none, - op2: Instruction.Operand = .none, - op3: Instruction.Operand = .none, - op4: Instruction.Operand = .none, -}) !void { - const err = Instruction.new(mnemonic, .{ - .op1 = args.op1, - .op2 = args.op2, - .op3 = args.op3, - .op4 = args.op4, - }); +fn invalidInstruction(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void { + const err = Instruction.new(mnemonic, args); try testing.expectError(error.InvalidInstruction, err); } @@ -1479,23 +1462,13 @@ test "invalid instruction" { try invalidInstruction(.push, .{ .op1 = .{ .imm = Immediate.u(0x1000000000000000) } }); } -fn cannotEncode(mnemonic: Instruction.Mnemonic, args: struct { - op1: Instruction.Operand = .none, - op2: Instruction.Operand = .none, - op3: Instruction.Operand = .none, - op4: Instruction.Operand = .none, -}) !void { - try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, .{ - .op1 = args.op1, - .op2 = args.op2, - .op3 = args.op3, - .op4 = args.op4, - })); +fn cannotEncode(mnemonic: Instruction.Mnemonic, args: Instruction.Init) !void { + try testing.expectError(error.CannotEncode, Instruction.new(mnemonic, args)); } test "cannot encode" { try cannotEncode(.@"test", .{ - .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12, .disp = 0 }) }, + .op1 = .{ .mem = Memory.sib(.byte, .{ .base = .r12 }) }, .op2 = .{ .reg = .ah }, }); try cannotEncode(.@"test", .{ diff --git a/src/arch/x86_64/encodings.zig b/src/arch/x86_64/encodings.zig index b008eb9f3e..23a125789b 100644 --- a/src/arch/x86_64/encodings.zig +++ b/src/arch/x86_64/encodings.zig @@ -4,618 +4,857 @@ const OpEn = Encoding.OpEn; const Op = Encoding.Op; const Mode = Encoding.Mode; -const opcode_len = u2; const modrm_ext = u3; -const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, opcode_len, u8, u8, u8, modrm_ext, Mode }; +const Entry = struct { Mnemonic, OpEn, Op, Op, Op, Op, []const u8, modrm_ext, Mode }; // TODO move this into a .zon file when Zig is capable of importing .zon files // zig fmt: off pub const table = &[_]Entry{ // General-purpose - .{ .adc, .zi, .al, .imm8, .none, .none, 1, 0x14, 0x00, 0x00, 0, .none }, - .{ .adc, .zi, .ax, .imm16, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none }, - .{ .adc, .zi, .eax, .imm32, .none, .none, 1, 0x15, 0x00, 0x00, 0, .none }, - .{ .adc, .zi, .rax, .imm32s, .none, .none, 1, 0x15, 0x00, 0x00, 0, .long }, - .{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .none }, - .{ .adc, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 2, .rex }, - .{ .adc, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none }, - .{ .adc, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 2, .none }, - .{ .adc, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 2, .long }, - .{ .adc, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none }, - .{ .adc, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .none }, - .{ .adc, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 2, .long }, - .{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .none }, - .{ .adc, .mr, .rm8, .r8, .none, .none, 1, 0x10, 0x00, 0x00, 0, .rex }, - .{ .adc, .mr, .rm16, .r16, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none }, - .{ .adc, .mr, .rm32, .r32, .none, .none, 1, 0x11, 0x00, 0x00, 0, .none }, - .{ .adc, .mr, .rm64, .r64, .none, .none, 1, 0x11, 0x00, 0x00, 0, .long }, - .{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .none }, - .{ .adc, .rm, .r8, .rm8, .none, .none, 1, 0x12, 0x00, 0x00, 0, .rex }, - .{ .adc, .rm, .r16, .rm16, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none }, - .{ .adc, .rm, .r32, .rm32, .none, .none, 1, 0x13, 0x00, 0x00, 0, .none }, - .{ .adc, .rm, .r64, .rm64, .none, .none, 1, 0x13, 0x00, 0x00, 0, .long }, + .{ .adc, .zi, .al, .imm8, .none, .none, &.{ 0x14 }, 0, .none }, + .{ .adc, .zi, .ax, .imm16, .none, .none, &.{ 0x15 }, 0, .none }, + .{ .adc, .zi, .eax, .imm32, .none, .none, &.{ 0x15 }, 0, .none }, + .{ .adc, .zi, .rax, .imm32s, .none, .none, &.{ 0x15 }, 0, .long }, + .{ .adc, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 2, .none }, + .{ .adc, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 2, .rex }, + .{ .adc, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 2, .none }, + .{ .adc, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 2, .none }, + .{ .adc, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 2, .long }, + .{ .adc, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 2, .none }, + .{ .adc, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 2, .none }, + .{ .adc, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 2, .long }, + .{ .adc, .mr, .rm8, .r8, .none, .none, &.{ 0x10 }, 0, .none }, + .{ .adc, .mr, .rm8, .r8, .none, .none, &.{ 0x10 }, 0, .rex }, + .{ .adc, .mr, .rm16, .r16, .none, .none, &.{ 0x11 }, 0, .none }, + .{ .adc, .mr, .rm32, .r32, .none, .none, &.{ 0x11 }, 0, .none }, + .{ .adc, .mr, .rm64, .r64, .none, .none, &.{ 0x11 }, 0, .long }, + .{ .adc, .rm, .r8, .rm8, .none, .none, &.{ 0x12 }, 0, .none }, + .{ .adc, .rm, .r8, .rm8, .none, .none, &.{ 0x12 }, 0, .rex }, + .{ .adc, .rm, .r16, .rm16, .none, .none, &.{ 0x13 }, 0, .none }, + .{ .adc, .rm, .r32, .rm32, .none, .none, &.{ 0x13 }, 0, .none }, + .{ .adc, .rm, .r64, .rm64, .none, .none, &.{ 0x13 }, 0, .long }, - .{ .add, .zi, .al, .imm8, .none, .none, 1, 0x04, 0x00, 0x00, 0, .none }, - .{ .add, .zi, .ax, .imm16, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none }, - .{ .add, .zi, .eax, .imm32, .none, .none, 1, 0x05, 0x00, 0x00, 0, .none }, - .{ .add, .zi, .rax, .imm32s, .none, .none, 1, 0x05, 0x00, 0x00, 0, .long }, - .{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .none }, - .{ .add, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 0, .rex }, - .{ .add, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none }, - .{ .add, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 0, .none }, - .{ .add, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 0, .long }, - .{ .add, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none }, - .{ .add, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .none }, - .{ .add, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 0, .long }, - .{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .none }, - .{ .add, .mr, .rm8, .r8, .none, .none, 1, 0x00, 0x00, 0x00, 0, .rex }, - .{ .add, .mr, .rm16, .r16, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none }, - .{ .add, .mr, .rm32, .r32, .none, .none, 1, 0x01, 0x00, 0x00, 0, .none }, - .{ .add, .mr, .rm64, .r64, .none, .none, 1, 0x01, 0x00, 0x00, 0, .long }, - .{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .none }, - .{ .add, .rm, .r8, .rm8, .none, .none, 1, 0x02, 0x00, 0x00, 0, .rex }, - .{ .add, .rm, .r16, .rm16, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none }, - .{ .add, .rm, .r32, .rm32, .none, .none, 1, 0x03, 0x00, 0x00, 0, .none }, - .{ .add, .rm, .r64, .rm64, .none, .none, 1, 0x03, 0x00, 0x00, 0, .long }, + .{ .add, .zi, .al, .imm8, .none, .none, &.{ 0x04 }, 0, .none }, + .{ .add, .zi, .ax, .imm16, .none, .none, &.{ 0x05 }, 0, .none }, + .{ .add, .zi, .eax, .imm32, .none, .none, &.{ 0x05 }, 0, .none }, + .{ .add, .zi, .rax, .imm32s, .none, .none, &.{ 0x05 }, 0, .long }, + .{ .add, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 0, .none }, + .{ .add, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 0, .rex }, + .{ .add, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 0, .none }, + .{ .add, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 0, .none }, + .{ .add, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 0, .long }, + .{ .add, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 0, .none }, + .{ .add, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 0, .none }, + .{ .add, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 0, .long }, + .{ .add, .mr, .rm8, .r8, .none, .none, &.{ 0x00 }, 0, .none }, + .{ .add, .mr, .rm8, .r8, .none, .none, &.{ 0x00 }, 0, .rex }, + .{ .add, .mr, .rm16, .r16, .none, .none, &.{ 0x01 }, 0, .none }, + .{ .add, .mr, .rm32, .r32, .none, .none, &.{ 0x01 }, 0, .none }, + .{ .add, .mr, .rm64, .r64, .none, .none, &.{ 0x01 }, 0, .long }, + .{ .add, .rm, .r8, .rm8, .none, .none, &.{ 0x02 }, 0, .none }, + .{ .add, .rm, .r8, .rm8, .none, .none, &.{ 0x02 }, 0, .rex }, + .{ .add, .rm, .r16, .rm16, .none, .none, &.{ 0x03 }, 0, .none }, + .{ .add, .rm, .r32, .rm32, .none, .none, &.{ 0x03 }, 0, .none }, + .{ .add, .rm, .r64, .rm64, .none, .none, &.{ 0x03 }, 0, .long }, - .{ .@"and", .zi, .al, .imm8, .none, .none, 1, 0x24, 0x00, 0x00, 0, .none }, - .{ .@"and", .zi, .ax, .imm16, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none }, - .{ .@"and", .zi, .eax, .imm32, .none, .none, 1, 0x25, 0x00, 0x00, 0, .none }, - .{ .@"and", .zi, .rax, .imm32s, .none, .none, 1, 0x25, 0x00, 0x00, 0, .long }, - .{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .none }, - .{ .@"and", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 4, .rex }, - .{ .@"and", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none }, - .{ .@"and", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 4, .none }, - .{ .@"and", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 4, .long }, - .{ .@"and", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none }, - .{ .@"and", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .none }, - .{ .@"and", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 4, .long }, - .{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .none }, - .{ .@"and", .mr, .rm8, .r8, .none, .none, 1, 0x20, 0x00, 0x00, 0, .rex }, - .{ .@"and", .mr, .rm16, .r16, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none }, - .{ .@"and", .mr, .rm32, .r32, .none, .none, 1, 0x21, 0x00, 0x00, 0, .none }, - .{ .@"and", .mr, .rm64, .r64, .none, .none, 1, 0x21, 0x00, 0x00, 0, .long }, - .{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .none }, - .{ .@"and", .rm, .r8, .rm8, .none, .none, 1, 0x22, 0x00, 0x00, 0, .rex }, - .{ .@"and", .rm, .r16, .rm16, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none }, - .{ .@"and", .rm, .r32, .rm32, .none, .none, 1, 0x23, 0x00, 0x00, 0, .none }, - .{ .@"and", .rm, .r64, .rm64, .none, .none, 1, 0x23, 0x00, 0x00, 0, .long }, + .{ .@"and", .zi, .al, .imm8, .none, .none, &.{ 0x24 }, 0, .none }, + .{ .@"and", .zi, .ax, .imm16, .none, .none, &.{ 0x25 }, 0, .none }, + .{ .@"and", .zi, .eax, .imm32, .none, .none, &.{ 0x25 }, 0, .none }, + .{ .@"and", .zi, .rax, .imm32s, .none, .none, &.{ 0x25 }, 0, .long }, + .{ .@"and", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 4, .none }, + .{ .@"and", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 4, .rex }, + .{ .@"and", .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 4, .none }, + .{ .@"and", .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 4, .none }, + .{ .@"and", .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 4, .long }, + .{ .@"and", .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 4, .none }, + .{ .@"and", .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 4, .none }, + .{ .@"and", .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 4, .long }, + .{ .@"and", .mr, .rm8, .r8, .none, .none, &.{ 0x20 }, 0, .none }, + .{ .@"and", .mr, .rm8, .r8, .none, .none, &.{ 0x20 }, 0, .rex }, + .{ .@"and", .mr, .rm16, .r16, .none, .none, &.{ 0x21 }, 0, .none }, + .{ .@"and", .mr, .rm32, .r32, .none, .none, &.{ 0x21 }, 0, .none }, + .{ .@"and", .mr, .rm64, .r64, .none, .none, &.{ 0x21 }, 0, .long }, + .{ .@"and", .rm, .r8, .rm8, .none, .none, &.{ 0x22 }, 0, .none }, + .{ .@"and", .rm, .r8, .rm8, .none, .none, &.{ 0x22 }, 0, .rex }, + .{ .@"and", .rm, .r16, .rm16, .none, .none, &.{ 0x23 }, 0, .none }, + .{ .@"and", .rm, .r32, .rm32, .none, .none, &.{ 0x23 }, 0, .none }, + .{ .@"and", .rm, .r64, .rm64, .none, .none, &.{ 0x23 }, 0, .long }, + + .{ .bsf, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbc }, 0, .none }, + .{ .bsf, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbc }, 0, .none }, + .{ .bsf, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbc }, 0, .long }, + + .{ .bsr, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xbd }, 0, .none }, + .{ .bsr, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xbd }, 0, .none }, + .{ .bsr, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xbd }, 0, .long }, + + .{ .bswap, .o, .r32, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .none }, + .{ .bswap, .o, .r64, .none, .none, .none, &.{ 0x0f, 0xc8 }, 0, .long }, + + .{ .bt, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xa3 }, 0, .none }, + .{ .bt, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xa3 }, 0, .none }, + .{ .bt, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xa3 }, 0, .long }, + .{ .bt, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .none }, + .{ .bt, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .none }, + .{ .bt, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 4, .long }, + + .{ .btc, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xbb }, 0, .none }, + .{ .btc, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xbb }, 0, .none }, + .{ .btc, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xbb }, 0, .long }, + .{ .btc, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .none }, + .{ .btc, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .none }, + .{ .btc, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 7, .long }, + + .{ .btr, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xb3 }, 0, .none }, + .{ .btr, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xb3 }, 0, .none }, + .{ .btr, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xb3 }, 0, .long }, + .{ .btr, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .none }, + .{ .btr, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .none }, + .{ .btr, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 6, .long }, + + .{ .bts, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xab }, 0, .none }, + .{ .bts, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xab }, 0, .none }, + .{ .bts, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xab }, 0, .long }, + .{ .bts, .mi, .rm16, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .none }, + .{ .bts, .mi, .rm32, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .none }, + .{ .bts, .mi, .rm64, .imm8, .none, .none, &.{ 0x0f, 0xba }, 5, .long }, // This is M encoding according to Intel, but D makes more sense here. - .{ .call, .d, .rel32, .none, .none, .none, 1, 0xe8, 0x00, 0x00, 0, .none }, - .{ .call, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 2, .none }, + .{ .call, .d, .rel32, .none, .none, .none, &.{ 0xe8 }, 0, .none }, + .{ .call, .m, .rm64, .none, .none, .none, &.{ 0xff }, 2, .none }, - .{ .cbw, .np, .o16, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none }, - .{ .cwde, .np, .o32, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .none }, - .{ .cdqe, .np, .o64, .none, .none, .none, 1, 0x98, 0x00, 0x00, 0, .long }, + .{ .cbw, .np, .o16, .none, .none, .none, &.{ 0x98 }, 0, .none }, + .{ .cwde, .np, .o32, .none, .none, .none, &.{ 0x98 }, 0, .none }, + .{ .cdqe, .np, .o64, .none, .none, .none, &.{ 0x98 }, 0, .long }, - .{ .cwd, .np, .o16, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none }, - .{ .cdq, .np, .o32, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .none }, - .{ .cqo, .np, .o64, .none, .none, .none, 1, 0x99, 0x00, 0x00, 0, .long }, + .{ .cwd, .np, .o16, .none, .none, .none, &.{ 0x99 }, 0, .none }, + .{ .cdq, .np, .o32, .none, .none, .none, &.{ 0x99 }, 0, .none }, + .{ .cqo, .np, .o64, .none, .none, .none, &.{ 0x99 }, 0, .long }, - .{ .cmova, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none }, - .{ .cmova, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none }, - .{ .cmova, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long }, - .{ .cmovae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none }, - .{ .cmovae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none }, - .{ .cmovae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long }, - .{ .cmovb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none }, - .{ .cmovb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none }, - .{ .cmovb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long }, - .{ .cmovbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none }, - .{ .cmovbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none }, - .{ .cmovbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long }, - .{ .cmovc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none }, - .{ .cmovc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none }, - .{ .cmovc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long }, - .{ .cmove, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none }, - .{ .cmove, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none }, - .{ .cmove, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long }, - .{ .cmovg, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none }, - .{ .cmovg, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none }, - .{ .cmovg, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long }, - .{ .cmovge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none }, - .{ .cmovge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none }, - .{ .cmovge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long }, - .{ .cmovl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none }, - .{ .cmovl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none }, - .{ .cmovl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long }, - .{ .cmovle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none }, - .{ .cmovle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none }, - .{ .cmovle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long }, - .{ .cmovna, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none }, - .{ .cmovna, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .none }, - .{ .cmovna, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x46, 0x00, 0, .long }, - .{ .cmovnae, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none }, - .{ .cmovnae, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .none }, - .{ .cmovnae, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x42, 0x00, 0, .long }, - .{ .cmovnb, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none }, - .{ .cmovnb, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none }, - .{ .cmovnb, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long }, - .{ .cmovnbe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none }, - .{ .cmovnbe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .none }, - .{ .cmovnbe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x47, 0x00, 0, .long }, - .{ .cmovnc, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none }, - .{ .cmovnc, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .none }, - .{ .cmovnc, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x43, 0x00, 0, .long }, - .{ .cmovne, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none }, - .{ .cmovne, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none }, - .{ .cmovne, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long }, - .{ .cmovng, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none }, - .{ .cmovng, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .none }, - .{ .cmovng, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4e, 0x00, 0, .long }, - .{ .cmovnge, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none }, - .{ .cmovnge, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .none }, - .{ .cmovnge, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4c, 0x00, 0, .long }, - .{ .cmovnl, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none }, - .{ .cmovnl, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .none }, - .{ .cmovnl, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4d, 0x00, 0, .long }, - .{ .cmovnle, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none }, - .{ .cmovnle, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .none }, - .{ .cmovnle, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4f, 0x00, 0, .long }, - .{ .cmovno, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none }, - .{ .cmovno, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .none }, - .{ .cmovno, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x41, 0x00, 0, .long }, - .{ .cmovnp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none }, - .{ .cmovnp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none }, - .{ .cmovnp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long }, - .{ .cmovns, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none }, - .{ .cmovns, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .none }, - .{ .cmovns, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x49, 0x00, 0, .long }, - .{ .cmovnz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none }, - .{ .cmovnz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .none }, - .{ .cmovnz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x45, 0x00, 0, .long }, - .{ .cmovo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none }, - .{ .cmovo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .none }, - .{ .cmovo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x40, 0x00, 0, .long }, - .{ .cmovp, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none }, - .{ .cmovp, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none }, - .{ .cmovp, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long }, - .{ .cmovpe, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none }, - .{ .cmovpe, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .none }, - .{ .cmovpe, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4a, 0x00, 0, .long }, - .{ .cmovpo, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none }, - .{ .cmovpo, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .none }, - .{ .cmovpo, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x4b, 0x00, 0, .long }, - .{ .cmovs, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none }, - .{ .cmovs, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .none }, - .{ .cmovs, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x48, 0x00, 0, .long }, - .{ .cmovz, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none }, - .{ .cmovz, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .none }, - .{ .cmovz, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0x44, 0x00, 0, .long }, + .{ .cmova, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x47 }, 0, .none }, + .{ .cmova, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x47 }, 0, .none }, + .{ .cmova, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x47 }, 0, .long }, + .{ .cmovae, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none }, + .{ .cmovae, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none }, + .{ .cmovae, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long }, + .{ .cmovb, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none }, + .{ .cmovb, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none }, + .{ .cmovb, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long }, + .{ .cmovbe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x46 }, 0, .none }, + .{ .cmovbe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x46 }, 0, .none }, + .{ .cmovbe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x46 }, 0, .long }, + .{ .cmovc, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none }, + .{ .cmovc, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none }, + .{ .cmovc, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long }, + .{ .cmove, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x44 }, 0, .none }, + .{ .cmove, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x44 }, 0, .none }, + .{ .cmove, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x44 }, 0, .long }, + .{ .cmovg, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4f }, 0, .none }, + .{ .cmovg, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4f }, 0, .none }, + .{ .cmovg, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4f }, 0, .long }, + .{ .cmovge, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4d }, 0, .none }, + .{ .cmovge, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4d }, 0, .none }, + .{ .cmovge, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4d }, 0, .long }, + .{ .cmovl, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4c }, 0, .none }, + .{ .cmovl, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4c }, 0, .none }, + .{ .cmovl, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4c }, 0, .long }, + .{ .cmovle, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4e }, 0, .none }, + .{ .cmovle, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4e }, 0, .none }, + .{ .cmovle, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4e }, 0, .long }, + .{ .cmovna, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x46 }, 0, .none }, + .{ .cmovna, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x46 }, 0, .none }, + .{ .cmovna, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x46 }, 0, .long }, + .{ .cmovnae, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x42 }, 0, .none }, + .{ .cmovnae, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x42 }, 0, .none }, + .{ .cmovnae, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x42 }, 0, .long }, + .{ .cmovnb, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none }, + .{ .cmovnb, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none }, + .{ .cmovnb, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long }, + .{ .cmovnbe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x47 }, 0, .none }, + .{ .cmovnbe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x47 }, 0, .none }, + .{ .cmovnbe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x47 }, 0, .long }, + .{ .cmovnc, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x43 }, 0, .none }, + .{ .cmovnc, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x43 }, 0, .none }, + .{ .cmovnc, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x43 }, 0, .long }, + .{ .cmovne, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x45 }, 0, .none }, + .{ .cmovne, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x45 }, 0, .none }, + .{ .cmovne, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x45 }, 0, .long }, + .{ .cmovng, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4e }, 0, .none }, + .{ .cmovng, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4e }, 0, .none }, + .{ .cmovng, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4e }, 0, .long }, + .{ .cmovnge, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4c }, 0, .none }, + .{ .cmovnge, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4c }, 0, .none }, + .{ .cmovnge, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4c }, 0, .long }, + .{ .cmovnl, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4d }, 0, .none }, + .{ .cmovnl, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4d }, 0, .none }, + .{ .cmovnl, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4d }, 0, .long }, + .{ .cmovnle, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4f }, 0, .none }, + .{ .cmovnle, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4f }, 0, .none }, + .{ .cmovnle, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4f }, 0, .long }, + .{ .cmovno, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x41 }, 0, .none }, + .{ .cmovno, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x41 }, 0, .none }, + .{ .cmovno, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x41 }, 0, .long }, + .{ .cmovnp, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4b }, 0, .none }, + .{ .cmovnp, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4b }, 0, .none }, + .{ .cmovnp, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4b }, 0, .long }, + .{ .cmovns, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x49 }, 0, .none }, + .{ .cmovns, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x49 }, 0, .none }, + .{ .cmovns, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x49 }, 0, .long }, + .{ .cmovnz, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x45 }, 0, .none }, + .{ .cmovnz, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x45 }, 0, .none }, + .{ .cmovnz, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x45 }, 0, .long }, + .{ .cmovo, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x40 }, 0, .none }, + .{ .cmovo, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x40 }, 0, .none }, + .{ .cmovo, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x40 }, 0, .long }, + .{ .cmovp, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4a }, 0, .none }, + .{ .cmovp, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4a }, 0, .none }, + .{ .cmovp, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4a }, 0, .long }, + .{ .cmovpe, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4a }, 0, .none }, + .{ .cmovpe, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4a }, 0, .none }, + .{ .cmovpe, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4a }, 0, .long }, + .{ .cmovpo, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x4b }, 0, .none }, + .{ .cmovpo, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x4b }, 0, .none }, + .{ .cmovpo, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x4b }, 0, .long }, + .{ .cmovs, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x48 }, 0, .none }, + .{ .cmovs, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x48 }, 0, .none }, + .{ .cmovs, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x48 }, 0, .long }, + .{ .cmovz, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0x44 }, 0, .none }, + .{ .cmovz, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0x44 }, 0, .none }, + .{ .cmovz, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0x44 }, 0, .long }, - .{ .cmp, .zi, .al, .imm8, .none, .none, 1, 0x3c, 0x00, 0x00, 0, .none }, - .{ .cmp, .zi, .ax, .imm16, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none }, - .{ .cmp, .zi, .eax, .imm32, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .none }, - .{ .cmp, .zi, .rax, .imm32s, .none, .none, 1, 0x3d, 0x00, 0x00, 0, .long }, - .{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .none }, - .{ .cmp, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 7, .rex }, - .{ .cmp, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none }, - .{ .cmp, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 7, .none }, - .{ .cmp, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 7, .long }, - .{ .cmp, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none }, - .{ .cmp, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .none }, - .{ .cmp, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 7, .long }, - .{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .none }, - .{ .cmp, .mr, .rm8, .r8, .none, .none, 1, 0x38, 0x00, 0x00, 0, .rex }, - .{ .cmp, .mr, .rm16, .r16, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none }, - .{ .cmp, .mr, .rm32, .r32, .none, .none, 1, 0x39, 0x00, 0x00, 0, .none }, - .{ .cmp, .mr, .rm64, .r64, .none, .none, 1, 0x39, 0x00, 0x00, 0, .long }, - .{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .none }, - .{ .cmp, .rm, .r8, .rm8, .none, .none, 1, 0x3a, 0x00, 0x00, 0, .rex }, - .{ .cmp, .rm, .r16, .rm16, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none }, - .{ .cmp, .rm, .r32, .rm32, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .none }, - .{ .cmp, .rm, .r64, .rm64, .none, .none, 1, 0x3b, 0x00, 0x00, 0, .long }, + .{ .cmp, .zi, .al, .imm8, .none, .none, &.{ 0x3c }, 0, .none }, + .{ .cmp, .zi, .ax, .imm16, .none, .none, &.{ 0x3d }, 0, .none }, + .{ .cmp, .zi, .eax, .imm32, .none, .none, &.{ 0x3d }, 0, .none }, + .{ .cmp, .zi, .rax, .imm32s, .none, .none, &.{ 0x3d }, 0, .long }, + .{ .cmp, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 7, .none }, + .{ .cmp, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 7, .rex }, + .{ .cmp, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 7, .none }, + .{ .cmp, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 7, .none }, + .{ .cmp, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 7, .long }, + .{ .cmp, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 7, .none }, + .{ .cmp, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 7, .none }, + .{ .cmp, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 7, .long }, + .{ .cmp, .mr, .rm8, .r8, .none, .none, &.{ 0x38 }, 0, .none }, + .{ .cmp, .mr, .rm8, .r8, .none, .none, &.{ 0x38 }, 0, .rex }, + .{ .cmp, .mr, .rm16, .r16, .none, .none, &.{ 0x39 }, 0, .none }, + .{ .cmp, .mr, .rm32, .r32, .none, .none, &.{ 0x39 }, 0, .none }, + .{ .cmp, .mr, .rm64, .r64, .none, .none, &.{ 0x39 }, 0, .long }, + .{ .cmp, .rm, .r8, .rm8, .none, .none, &.{ 0x3a }, 0, .none }, + .{ .cmp, .rm, .r8, .rm8, .none, .none, &.{ 0x3a }, 0, .rex }, + .{ .cmp, .rm, .r16, .rm16, .none, .none, &.{ 0x3b }, 0, .none }, + .{ .cmp, .rm, .r32, .rm32, .none, .none, &.{ 0x3b }, 0, .none }, + .{ .cmp, .rm, .r64, .rm64, .none, .none, &.{ 0x3b }, 0, .long }, - .{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .none }, - .{ .div, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 6, .rex }, - .{ .div, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none }, - .{ .div, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .none }, - .{ .div, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 6, .long }, + .{ .cmps, .np, .m8, .m8, .none, .none, &.{ 0xa6 }, 0, .none }, + .{ .cmps, .np, .m16, .m16, .none, .none, &.{ 0xa7 }, 0, .none }, + .{ .cmps, .np, .m32, .m32, .none, .none, &.{ 0xa7 }, 0, .none }, + .{ .cmps, .np, .m64, .m64, .none, .none, &.{ 0xa7 }, 0, .long }, + .{ .cmpsb, .np, .none, .none, .none, .none, &.{ 0xa6 }, 0, .none }, + .{ .cmpsw, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .short }, + .{ .cmpsd, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .none }, + .{ .cmpsq, .np, .none, .none, .none, .none, &.{ 0xa7 }, 0, .long }, - .{ .fisttp, .m, .m16, .none, .none, .none, 1, 0xdf, 0x00, 0x00, 1, .fpu }, - .{ .fisttp, .m, .m32, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 1, .fpu }, - .{ .fisttp, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 1, .fpu }, + .{ .cmpxchg, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xb0 }, 0, .none }, + .{ .cmpxchg, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xb0 }, 0, .rex }, + .{ .cmpxchg, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex }, + .{ .cmpxchg, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xb1 }, 0, .rex }, + .{ .cmpxchg, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xb1 }, 0, .long }, - .{ .fld, .m, .m32, .none, .none, .none, 1, 0xd9, 0x00, 0x00, 0, .fpu }, - .{ .fld, .m, .m64, .none, .none, .none, 1, 0xdd, 0x00, 0x00, 0, .fpu }, - .{ .fld, .m, .m80, .none, .none, .none, 1, 0xdb, 0x00, 0x00, 5, .fpu }, + .{ .cmpxchg8b , .m, .m64, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .none }, + .{ .cmpxchg16b, .m, .m128, .none, .none, .none, &.{ 0x0f, 0xc7 }, 1, .long }, - .{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .none }, - .{ .idiv, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 7, .rex }, - .{ .idiv, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none }, - .{ .idiv, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .none }, - .{ .idiv, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 7, .long }, + .{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .none }, + .{ .div, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 6, .rex }, + .{ .div, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 6, .none }, + .{ .div, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 6, .none }, + .{ .div, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 6, .long }, - .{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .none }, - .{ .imul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 5, .rex }, - .{ .imul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none }, - .{ .imul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .none }, - .{ .imul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 5, .long }, - .{ .imul, .rm, .r16, .rm16, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none }, - .{ .imul, .rm, .r32, .rm32, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .none }, - .{ .imul, .rm, .r64, .rm64, .none, .none, 2, 0x0f, 0xaf, 0x00, 0, .long }, - .{ .imul, .rmi, .r16, .rm16, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none }, - .{ .imul, .rmi, .r32, .rm32, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .none }, - .{ .imul, .rmi, .r64, .rm64, .imm8s, .none, 1, 0x6b, 0x00, 0x00, 0, .long }, - .{ .imul, .rmi, .r16, .rm16, .imm16, .none, 1, 0x69, 0x00, 0x00, 0, .none }, - .{ .imul, .rmi, .r32, .rm32, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .none }, - .{ .imul, .rmi, .r64, .rm64, .imm32, .none, 1, 0x69, 0x00, 0x00, 0, .long }, + .{ .fisttp, .m, .m16, .none, .none, .none, &.{ 0xdf }, 1, .fpu }, + .{ .fisttp, .m, .m32, .none, .none, .none, &.{ 0xdb }, 1, .fpu }, + .{ .fisttp, .m, .m64, .none, .none, .none, &.{ 0xdd }, 1, .fpu }, - .{ .int3, .np, .none, .none, .none, .none, 1, 0xcc, 0x00, 0x00, 0, .none }, + .{ .fld, .m, .m32, .none, .none, .none, &.{ 0xd9 }, 0, .fpu }, + .{ .fld, .m, .m64, .none, .none, .none, &.{ 0xdd }, 0, .fpu }, + .{ .fld, .m, .m80, .none, .none, .none, &.{ 0xdb }, 5, .fpu }, - .{ .ja, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none }, - .{ .jae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none }, - .{ .jb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none }, - .{ .jbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none }, - .{ .jc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none }, - .{ .jrcxz, .d, .rel32, .none, .none, .none, 1, 0xe3, 0x00, 0x00, 0, .none }, - .{ .je, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none }, - .{ .jg, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none }, - .{ .jge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none }, - .{ .jl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none }, - .{ .jle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none }, - .{ .jna, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x86, 0x00, 0, .none }, - .{ .jnae, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x82, 0x00, 0, .none }, - .{ .jnb, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none }, - .{ .jnbe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x87, 0x00, 0, .none }, - .{ .jnc, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x83, 0x00, 0, .none }, - .{ .jne, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none }, - .{ .jng, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8e, 0x00, 0, .none }, - .{ .jnge, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8c, 0x00, 0, .none }, - .{ .jnl, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8d, 0x00, 0, .none }, - .{ .jnle, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8f, 0x00, 0, .none }, - .{ .jno, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x81, 0x00, 0, .none }, - .{ .jnp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none }, - .{ .jns, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x89, 0x00, 0, .none }, - .{ .jnz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x85, 0x00, 0, .none }, - .{ .jo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x80, 0x00, 0, .none }, - .{ .jp, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none }, - .{ .jpe, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8a, 0x00, 0, .none }, - .{ .jpo, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x8b, 0x00, 0, .none }, - .{ .js, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x88, 0x00, 0, .none }, - .{ .jz, .d, .rel32, .none, .none, .none, 2, 0x0f, 0x84, 0x00, 0, .none }, + .{ .idiv, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 7, .none }, + .{ .idiv, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 7, .rex }, + .{ .idiv, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 7, .none }, + .{ .idiv, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 7, .none }, + .{ .idiv, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 7, .long }, - .{ .jmp, .d, .rel32, .none, .none, .none, 1, 0xe9, 0x00, 0x00, 0, .none }, - .{ .jmp, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 4, .none }, + .{ .imul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 5, .none }, + .{ .imul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 5, .rex }, + .{ .imul, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 5, .none }, + .{ .imul, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 5, .none }, + .{ .imul, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 5, .long }, + .{ .imul, .rm, .r16, .rm16, .none, .none, &.{ 0x0f, 0xaf }, 0, .none }, + .{ .imul, .rm, .r32, .rm32, .none, .none, &.{ 0x0f, 0xaf }, 0, .none }, + .{ .imul, .rm, .r64, .rm64, .none, .none, &.{ 0x0f, 0xaf }, 0, .long }, + .{ .imul, .rmi, .r16, .rm16, .imm8s, .none, &.{ 0x6b }, 0, .none }, + .{ .imul, .rmi, .r32, .rm32, .imm8s, .none, &.{ 0x6b }, 0, .none }, + .{ .imul, .rmi, .r64, .rm64, .imm8s, .none, &.{ 0x6b }, 0, .long }, + .{ .imul, .rmi, .r16, .rm16, .imm16, .none, &.{ 0x69 }, 0, .none }, + .{ .imul, .rmi, .r32, .rm32, .imm32, .none, &.{ 0x69 }, 0, .none }, + .{ .imul, .rmi, .r64, .rm64, .imm32, .none, &.{ 0x69 }, 0, .long }, - .{ .lea, .rm, .r16, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none }, - .{ .lea, .rm, .r32, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .none }, - .{ .lea, .rm, .r64, .m, .none, .none, 1, 0x8d, 0x00, 0x00, 0, .long }, + .{ .int3, .np, .none, .none, .none, .none, &.{ 0xcc }, 0, .none }, - .{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .none }, - .{ .mov, .mr, .rm8, .r8, .none, .none, 1, 0x88, 0x00, 0x00, 0, .rex }, - .{ .mov, .mr, .rm16, .r16, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none }, - .{ .mov, .mr, .rm32, .r32, .none, .none, 1, 0x89, 0x00, 0x00, 0, .none }, - .{ .mov, .mr, .rm64, .r64, .none, .none, 1, 0x89, 0x00, 0x00, 0, .long }, - .{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .none }, - .{ .mov, .rm, .r8, .rm8, .none, .none, 1, 0x8a, 0x00, 0x00, 0, .rex }, - .{ .mov, .rm, .r16, .rm16, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none }, - .{ .mov, .rm, .r32, .rm32, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .none }, - .{ .mov, .rm, .r64, .rm64, .none, .none, 1, 0x8b, 0x00, 0x00, 0, .long }, - .{ .mov, .mr, .rm16, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .none }, - .{ .mov, .mr, .rm64, .sreg, .none, .none, 1, 0x8c, 0x00, 0x00, 0, .long }, - .{ .mov, .rm, .sreg, .rm16, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .none }, - .{ .mov, .rm, .sreg, .rm64, .none, .none, 1, 0x8e, 0x00, 0x00, 0, .long }, - .{ .mov, .fd, .al, .moffs, .none, .none, 1, 0xa0, 0x00, 0x00, 0, .none }, - .{ .mov, .fd, .ax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none }, - .{ .mov, .fd, .eax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .none }, - .{ .mov, .fd, .rax, .moffs, .none, .none, 1, 0xa1, 0x00, 0x00, 0, .long }, - .{ .mov, .td, .moffs, .al, .none, .none, 1, 0xa2, 0x00, 0x00, 0, .none }, - .{ .mov, .td, .moffs, .ax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none }, - .{ .mov, .td, .moffs, .eax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .none }, - .{ .mov, .td, .moffs, .rax, .none, .none, 1, 0xa3, 0x00, 0x00, 0, .long }, - .{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .none }, - .{ .mov, .oi, .r8, .imm8, .none, .none, 1, 0xb0, 0x00, 0x00, 0, .rex }, - .{ .mov, .oi, .r16, .imm16, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none }, - .{ .mov, .oi, .r32, .imm32, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .none }, - .{ .mov, .oi, .r64, .imm64, .none, .none, 1, 0xb8, 0x00, 0x00, 0, .long }, - .{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .none }, - .{ .mov, .mi, .rm8, .imm8, .none, .none, 1, 0xc6, 0x00, 0x00, 0, .rex }, - .{ .mov, .mi, .rm16, .imm16, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none }, - .{ .mov, .mi, .rm32, .imm32, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .none }, - .{ .mov, .mi, .rm64, .imm32s, .none, .none, 1, 0xc7, 0x00, 0x00, 0, .long }, + .{ .ja, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x87 }, 0, .none }, + .{ .jae, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none }, + .{ .jb, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none }, + .{ .jbe, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x86 }, 0, .none }, + .{ .jc, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none }, + .{ .jrcxz, .d, .rel32, .none, .none, .none, &.{ 0xe3 }, 0, .none }, + .{ .je, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x84 }, 0, .none }, + .{ .jg, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8f }, 0, .none }, + .{ .jge, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8d }, 0, .none }, + .{ .jl, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8c }, 0, .none }, + .{ .jle, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8e }, 0, .none }, + .{ .jna, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x86 }, 0, .none }, + .{ .jnae, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x82 }, 0, .none }, + .{ .jnb, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none }, + .{ .jnbe, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x87 }, 0, .none }, + .{ .jnc, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x83 }, 0, .none }, + .{ .jne, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x85 }, 0, .none }, + .{ .jng, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8e }, 0, .none }, + .{ .jnge, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8c }, 0, .none }, + .{ .jnl, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8d }, 0, .none }, + .{ .jnle, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8f }, 0, .none }, + .{ .jno, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x81 }, 0, .none }, + .{ .jnp, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8b }, 0, .none }, + .{ .jns, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x89 }, 0, .none }, + .{ .jnz, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x85 }, 0, .none }, + .{ .jo, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x80 }, 0, .none }, + .{ .jp, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8a }, 0, .none }, + .{ .jpe, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8a }, 0, .none }, + .{ .jpo, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x8b }, 0, .none }, + .{ .js, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x88 }, 0, .none }, + .{ .jz, .d, .rel32, .none, .none, .none, &.{ 0x0f, 0x84 }, 0, .none }, - .{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none }, - .{ .movsx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex }, - .{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .none }, - .{ .movsx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .rex }, - .{ .movsx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xbe, 0x00, 0, .long }, - .{ .movsx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .none }, - .{ .movsx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xbf, 0x00, 0, .long }, + .{ .jmp, .d, .rel32, .none, .none, .none, &.{ 0xe9 }, 0, .none }, + .{ .jmp, .m, .rm64, .none, .none, .none, &.{ 0xff }, 4, .none }, + + .{ .lea, .rm, .r16, .m, .none, .none, &.{ 0x8d }, 0, .none }, + .{ .lea, .rm, .r32, .m, .none, .none, &.{ 0x8d }, 0, .none }, + .{ .lea, .rm, .r64, .m, .none, .none, &.{ 0x8d }, 0, .long }, + + .{ .lfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xe8 }, 0, .none }, + + .{ .lods, .np, .m8, .none, .none, .none, &.{ 0xac }, 0, .none }, + .{ .lods, .np, .m16, .none, .none, .none, &.{ 0xad }, 0, .none }, + .{ .lods, .np, .m32, .none, .none, .none, &.{ 0xad }, 0, .none }, + .{ .lods, .np, .m64, .none, .none, .none, &.{ 0xad }, 0, .long }, + .{ .lodsb, .np, .none, .none, .none, .none, &.{ 0xac }, 0, .none }, + .{ .lodsw, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .short }, + .{ .lodsd, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .none }, + .{ .lodsq, .np, .none, .none, .none, .none, &.{ 0xad }, 0, .long }, + + .{ .lzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none }, + .{ .lzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .none }, + .{ .lzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbd }, 0, .long }, + + .{ .mfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf0 }, 0, .none }, + + .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .none }, + .{ .mov, .mr, .rm8, .r8, .none, .none, &.{ 0x88 }, 0, .rex }, + .{ .mov, .mr, .rm16, .r16, .none, .none, &.{ 0x89 }, 0, .none }, + .{ .mov, .mr, .rm32, .r32, .none, .none, &.{ 0x89 }, 0, .none }, + .{ .mov, .mr, .rm64, .r64, .none, .none, &.{ 0x89 }, 0, .long }, + .{ .mov, .rm, .r8, .rm8, .none, .none, &.{ 0x8a }, 0, .none }, + .{ .mov, .rm, .r8, .rm8, .none, .none, &.{ 0x8a }, 0, .rex }, + .{ .mov, .rm, .r16, .rm16, .none, .none, &.{ 0x8b }, 0, .none }, + .{ .mov, .rm, .r32, .rm32, .none, .none, &.{ 0x8b }, 0, .none }, + .{ .mov, .rm, .r64, .rm64, .none, .none, &.{ 0x8b }, 0, .long }, + .{ .mov, .mr, .rm16, .sreg, .none, .none, &.{ 0x8c }, 0, .none }, + .{ .mov, .mr, .rm64, .sreg, .none, .none, &.{ 0x8c }, 0, .long }, + .{ .mov, .rm, .sreg, .rm16, .none, .none, &.{ 0x8e }, 0, .none }, + .{ .mov, .rm, .sreg, .rm64, .none, .none, &.{ 0x8e }, 0, .long }, + .{ .mov, .fd, .al, .moffs, .none, .none, &.{ 0xa0 }, 0, .none }, + .{ .mov, .fd, .ax, .moffs, .none, .none, &.{ 0xa1 }, 0, .none }, + .{ .mov, .fd, .eax, .moffs, .none, .none, &.{ 0xa1 }, 0, .none }, + .{ .mov, .fd, .rax, .moffs, .none, .none, &.{ 0xa1 }, 0, .long }, + .{ .mov, .td, .moffs, .al, .none, .none, &.{ 0xa2 }, 0, .none }, + .{ .mov, .td, .moffs, .ax, .none, .none, &.{ 0xa3 }, 0, .none }, + .{ .mov, .td, .moffs, .eax, .none, .none, &.{ 0xa3 }, 0, .none }, + .{ .mov, .td, .moffs, .rax, .none, .none, &.{ 0xa3 }, 0, .long }, + .{ .mov, .oi, .r8, .imm8, .none, .none, &.{ 0xb0 }, 0, .none }, + .{ .mov, .oi, .r8, .imm8, .none, .none, &.{ 0xb0 }, 0, .rex }, + .{ .mov, .oi, .r16, .imm16, .none, .none, &.{ 0xb8 }, 0, .none }, + .{ .mov, .oi, .r32, .imm32, .none, .none, &.{ 0xb8 }, 0, .none }, + .{ .mov, .oi, .r64, .imm64, .none, .none, &.{ 0xb8 }, 0, .long }, + .{ .mov, .mi, .rm8, .imm8, .none, .none, &.{ 0xc6 }, 0, .none }, + .{ .mov, .mi, .rm8, .imm8, .none, .none, &.{ 0xc6 }, 0, .rex }, + .{ .mov, .mi, .rm16, .imm16, .none, .none, &.{ 0xc7 }, 0, .none }, + .{ .mov, .mi, .rm32, .imm32, .none, .none, &.{ 0xc7 }, 0, .none }, + .{ .mov, .mi, .rm64, .imm32s, .none, .none, &.{ 0xc7 }, 0, .long }, + + .{ .movbe, .rm, .r16, .m16, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none }, + .{ .movbe, .rm, .r32, .m32, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .none }, + .{ .movbe, .rm, .r64, .m64, .none, .none, &.{ 0x0f, 0x38, 0xf0 }, 0, .long }, + .{ .movbe, .mr, .m16, .r16, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none }, + .{ .movbe, .mr, .m32, .r32, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .none }, + .{ .movbe, .mr, .m64, .r64, .none, .none, &.{ 0x0f, 0x38, 0xf1 }, 0, .long }, + + .{ .movs, .np, .m8, .m8, .none, .none, &.{ 0xa4 }, 0, .none }, + .{ .movs, .np, .m16, .m16, .none, .none, &.{ 0xa5 }, 0, .none }, + .{ .movs, .np, .m32, .m32, .none, .none, &.{ 0xa5 }, 0, .none }, + .{ .movs, .np, .m64, .m64, .none, .none, &.{ 0xa5 }, 0, .long }, + .{ .movsb, .np, .none, .none, .none, .none, &.{ 0xa4 }, 0, .none }, + .{ .movsw, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .short }, + .{ .movsd, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .none }, + .{ .movsq, .np, .none, .none, .none, .none, &.{ 0xa5 }, 0, .long }, + + .{ .movsx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .none }, + .{ .movsx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .rex }, + .{ .movsx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .none }, + .{ .movsx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .rex }, + .{ .movsx, .rm, .r64, .rm8, .none, .none, &.{ 0x0f, 0xbe }, 0, .long }, + .{ .movsx, .rm, .r32, .rm16, .none, .none, &.{ 0x0f, 0xbf }, 0, .none }, + .{ .movsx, .rm, .r64, .rm16, .none, .none, &.{ 0x0f, 0xbf }, 0, .long }, // This instruction is discouraged. - .{ .movsxd, .rm, .r32, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .none }, - .{ .movsxd, .rm, .r64, .rm32, .none, .none, 1, 0x63, 0x00, 0x00, 0, .long }, + .{ .movsxd, .rm, .r32, .rm32, .none, .none, &.{ 0x63 }, 0, .none }, + .{ .movsxd, .rm, .r64, .rm32, .none, .none, &.{ 0x63 }, 0, .long }, - .{ .movzx, .rm, .r16, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none }, - .{ .movzx, .rm, .r32, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .none }, - .{ .movzx, .rm, .r64, .rm8, .none, .none, 2, 0x0f, 0xb6, 0x00, 0, .long }, - .{ .movzx, .rm, .r32, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .none }, - .{ .movzx, .rm, .r64, .rm16, .none, .none, 2, 0x0f, 0xb7, 0x00, 0, .long }, + .{ .movzx, .rm, .r16, .rm8, .none, .none, &.{ 0x0f, 0xb6 }, 0, .none }, + .{ .movzx, .rm, .r32, .rm8, .none, .none, &.{ 0x0f, 0xb6 }, 0, .none }, + .{ .movzx, .rm, .r64, .rm8, .none, .none, &.{ 0x0f, 0xb6 }, 0, .long }, + .{ .movzx, .rm, .r32, .rm16, .none, .none, &.{ 0x0f, 0xb7 }, 0, .none }, + .{ .movzx, .rm, .r64, .rm16, .none, .none, &.{ 0x0f, 0xb7 }, 0, .long }, - .{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .none }, - .{ .mul, .m, .rm8, .none, .none, .none, 1, 0xf6, 0x00, 0x00, 4, .rex }, - .{ .mul, .m, .rm16, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none }, - .{ .mul, .m, .rm32, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .none }, - .{ .mul, .m, .rm64, .none, .none, .none, 1, 0xf7, 0x00, 0x00, 4, .long }, + .{ .mul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 4, .none }, + .{ .mul, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 4, .rex }, + .{ .mul, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 4, .none }, + .{ .mul, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 4, .none }, + .{ .mul, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 4, .long }, - .{ .nop, .np, .none, .none, .none, .none, 1, 0x90, 0x00, 0x00, 0, .none }, + .{ .neg, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 3, .none }, + .{ .neg, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 3, .rex }, + .{ .neg, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 3, .none }, + .{ .neg, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 3, .none }, + .{ .neg, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 3, .long }, - .{ .@"or", .zi, .al, .imm8, .none, .none, 1, 0x0c, 0x00, 0x00, 0, .none }, - .{ .@"or", .zi, .ax, .imm16, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none }, - .{ .@"or", .zi, .eax, .imm32, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .none }, - .{ .@"or", .zi, .rax, .imm32s, .none, .none, 1, 0x0d, 0x00, 0x00, 0, .long }, - .{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .none }, - .{ .@"or", .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 1, .rex }, - .{ .@"or", .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none }, - .{ .@"or", .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 1, .none }, - .{ .@"or", .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 1, .long }, - .{ .@"or", .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none }, - .{ .@"or", .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .none }, - .{ .@"or", .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 1, .long }, - .{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .none }, - .{ .@"or", .mr, .rm8, .r8, .none, .none, 1, 0x08, 0x00, 0x00, 0, .rex }, - .{ .@"or", .mr, .rm16, .r16, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none }, - .{ .@"or", .mr, .rm32, .r32, .none, .none, 1, 0x09, 0x00, 0x00, 0, .none }, - .{ .@"or", .mr, .rm64, .r64, .none, .none, 1, 0x09, 0x00, 0x00, 0, .long }, - .{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .none }, - .{ .@"or", .rm, .r8, .rm8, .none, .none, 1, 0x0a, 0x00, 0x00, 0, .rex }, - .{ .@"or", .rm, .r16, .rm16, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none }, - .{ .@"or", .rm, .r32, .rm32, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .none }, - .{ .@"or", .rm, .r64, .rm64, .none, .none, 1, 0x0b, 0x00, 0x00, 0, .long }, + .{ .nop, .np, .none, .none, .none, .none, &.{ 0x90 }, 0, .none }, - .{ .pop, .o, .r16, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none }, - .{ .pop, .o, .r64, .none, .none, .none, 1, 0x58, 0x00, 0x00, 0, .none }, - .{ .pop, .m, .rm16, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none }, - .{ .pop, .m, .rm64, .none, .none, .none, 1, 0x8f, 0x00, 0x00, 0, .none }, + .{ .not, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 2, .none }, + .{ .not, .m, .rm8, .none, .none, .none, &.{ 0xf6 }, 2, .rex }, + .{ .not, .m, .rm16, .none, .none, .none, &.{ 0xf7 }, 2, .none }, + .{ .not, .m, .rm32, .none, .none, .none, &.{ 0xf7 }, 2, .none }, + .{ .not, .m, .rm64, .none, .none, .none, &.{ 0xf7 }, 2, .long }, - .{ .push, .o, .r16, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none }, - .{ .push, .o, .r64, .none, .none, .none, 1, 0x50, 0x00, 0x00, 0, .none }, - .{ .push, .m, .rm16, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none }, - .{ .push, .m, .rm64, .none, .none, .none, 1, 0xff, 0x00, 0x00, 6, .none }, - .{ .push, .i, .imm8, .none, .none, .none, 1, 0x6a, 0x00, 0x00, 0, .none }, - .{ .push, .i, .imm16, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none }, - .{ .push, .i, .imm32, .none, .none, .none, 1, 0x68, 0x00, 0x00, 0, .none }, + .{ .@"or", .zi, .al, .imm8, .none, .none, &.{ 0x0c }, 0, .none }, + .{ .@"or", .zi, .ax, .imm16, .none, .none, &.{ 0x0d }, 0, .none }, + .{ .@"or", .zi, .eax, .imm32, .none, .none, &.{ 0x0d }, 0, .none }, + .{ .@"or", .zi, .rax, .imm32s, .none, .none, &.{ 0x0d }, 0, .long }, + .{ .@"or", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 1, .none }, + .{ .@"or", .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 1, .rex }, + .{ .@"or", .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 1, .none }, + .{ .@"or", .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 1, .none }, + .{ .@"or", .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 1, .long }, + .{ .@"or", .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 1, .none }, + .{ .@"or", .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 1, .none }, + .{ .@"or", .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 1, .long }, + .{ .@"or", .mr, .rm8, .r8, .none, .none, &.{ 0x08 }, 0, .none }, + .{ .@"or", .mr, .rm8, .r8, .none, .none, &.{ 0x08 }, 0, .rex }, + .{ .@"or", .mr, .rm16, .r16, .none, .none, &.{ 0x09 }, 0, .none }, + .{ .@"or", .mr, .rm32, .r32, .none, .none, &.{ 0x09 }, 0, .none }, + .{ .@"or", .mr, .rm64, .r64, .none, .none, &.{ 0x09 }, 0, .long }, + .{ .@"or", .rm, .r8, .rm8, .none, .none, &.{ 0x0a }, 0, .none }, + .{ .@"or", .rm, .r8, .rm8, .none, .none, &.{ 0x0a }, 0, .rex }, + .{ .@"or", .rm, .r16, .rm16, .none, .none, &.{ 0x0b }, 0, .none }, + .{ .@"or", .rm, .r32, .rm32, .none, .none, &.{ 0x0b }, 0, .none }, + .{ .@"or", .rm, .r64, .rm64, .none, .none, &.{ 0x0b }, 0, .long }, - .{ .ret, .np, .none, .none, .none, .none, 1, 0xc3, 0x00, 0x00, 0, .none }, + .{ .pop, .o, .r16, .none, .none, .none, &.{ 0x58 }, 0, .none }, + .{ .pop, .o, .r64, .none, .none, .none, &.{ 0x58 }, 0, .none }, + .{ .pop, .m, .rm16, .none, .none, .none, &.{ 0x8f }, 0, .none }, + .{ .pop, .m, .rm64, .none, .none, .none, &.{ 0x8f }, 0, .none }, - .{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none }, - .{ .sal, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex }, - .{ .sal, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none }, - .{ .sal, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none }, - .{ .sal, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long }, - .{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none }, - .{ .sal, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex }, - .{ .sal, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none }, - .{ .sal, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none }, - .{ .sal, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long }, - .{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none }, - .{ .sal, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex }, - .{ .sal, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none }, - .{ .sal, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none }, - .{ .sal, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long }, + .{ .popcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none }, + .{ .popcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .none }, + .{ .popcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xb8 }, 0, .long }, - .{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .none }, - .{ .sar, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 7, .rex }, - .{ .sar, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none }, - .{ .sar, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .none }, - .{ .sar, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 7, .long }, - .{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .none }, - .{ .sar, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 7, .rex }, - .{ .sar, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none }, - .{ .sar, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .none }, - .{ .sar, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 7, .long }, - .{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .none }, - .{ .sar, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 7, .rex }, - .{ .sar, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none }, - .{ .sar, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .none }, - .{ .sar, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 7, .long }, + .{ .push, .o, .r16, .none, .none, .none, &.{ 0x50 }, 0, .none }, + .{ .push, .o, .r64, .none, .none, .none, &.{ 0x50 }, 0, .none }, + .{ .push, .m, .rm16, .none, .none, .none, &.{ 0xff }, 6, .none }, + .{ .push, .m, .rm64, .none, .none, .none, &.{ 0xff }, 6, .none }, + .{ .push, .i, .imm8, .none, .none, .none, &.{ 0x6a }, 0, .none }, + .{ .push, .i, .imm16, .none, .none, .none, &.{ 0x68 }, 0, .none }, + .{ .push, .i, .imm32, .none, .none, .none, &.{ 0x68 }, 0, .none }, - .{ .sbb, .zi, .al, .imm8, .none, .none, 1, 0x1c, 0x00, 0x00, 0, .none }, - .{ .sbb, .zi, .ax, .imm16, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none }, - .{ .sbb, .zi, .eax, .imm32, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .none }, - .{ .sbb, .zi, .rax, .imm32s, .none, .none, 1, 0x1d, 0x00, 0x00, 0, .long }, - .{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .none }, - .{ .sbb, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 3, .rex }, - .{ .sbb, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none }, - .{ .sbb, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 3, .none }, - .{ .sbb, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 3, .long }, - .{ .sbb, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none }, - .{ .sbb, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .none }, - .{ .sbb, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 3, .long }, - .{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .none }, - .{ .sbb, .mr, .rm8, .r8, .none, .none, 1, 0x18, 0x00, 0x00, 0, .rex }, - .{ .sbb, .mr, .rm16, .r16, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none }, - .{ .sbb, .mr, .rm32, .r32, .none, .none, 1, 0x19, 0x00, 0x00, 0, .none }, - .{ .sbb, .mr, .rm64, .r64, .none, .none, 1, 0x19, 0x00, 0x00, 0, .long }, - .{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .none }, - .{ .sbb, .rm, .r8, .rm8, .none, .none, 1, 0x1a, 0x00, 0x00, 0, .rex }, - .{ .sbb, .rm, .r16, .rm16, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none }, - .{ .sbb, .rm, .r32, .rm32, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .none }, - .{ .sbb, .rm, .r64, .rm64, .none, .none, 1, 0x1b, 0x00, 0x00, 0, .long }, + .{ .ret, .np, .none, .none, .none, .none, &.{ 0xc3 }, 0, .none }, - .{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none }, - .{ .seta, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex }, - .{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none }, - .{ .setae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex }, - .{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none }, - .{ .setb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex }, - .{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none }, - .{ .setbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex }, - .{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none }, - .{ .setc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex }, - .{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none }, - .{ .sete, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex }, - .{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none }, - .{ .setg, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex }, - .{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none }, - .{ .setge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex }, - .{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none }, - .{ .setl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex }, - .{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none }, - .{ .setle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex }, - .{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .none }, - .{ .setna, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x96, 0x00, 0, .rex }, - .{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .none }, - .{ .setnae, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x92, 0x00, 0, .rex }, - .{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none }, - .{ .setnb, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex }, - .{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .none }, - .{ .setnbe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x97, 0x00, 0, .rex }, - .{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .none }, - .{ .setnc, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x93, 0x00, 0, .rex }, - .{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none }, - .{ .setne, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex }, - .{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .none }, - .{ .setng, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9e, 0x00, 0, .rex }, - .{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .none }, - .{ .setnge, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9c, 0x00, 0, .rex }, - .{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .none }, - .{ .setnl, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9d, 0x00, 0, .rex }, - .{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .none }, - .{ .setnle, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9f, 0x00, 0, .rex }, - .{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .none }, - .{ .setno, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x91, 0x00, 0, .rex }, - .{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none }, - .{ .setnp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex }, - .{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .none }, - .{ .setns, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x99, 0x00, 0, .rex }, - .{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .none }, - .{ .setnz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x95, 0x00, 0, .rex }, - .{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .none }, - .{ .seto, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x90, 0x00, 0, .rex }, - .{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none }, - .{ .setp, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex }, - .{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .none }, - .{ .setpe, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9a, 0x00, 0, .rex }, - .{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .none }, - .{ .setpo, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x9b, 0x00, 0, .rex }, - .{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .none }, - .{ .sets, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x98, 0x00, 0, .rex }, - .{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .none }, - .{ .setz, .m, .rm8, .none, .none, .none, 2, 0x0f, 0x94, 0x00, 0, .rex }, + .{ .rcl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 2, .none }, + .{ .rcl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 2, .rex }, + .{ .rcl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 2, .none }, + .{ .rcl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 2, .rex }, + .{ .rcl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 2, .none }, + .{ .rcl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 2, .rex }, + .{ .rcl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 2, .none }, + .{ .rcl, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 2, .none }, + .{ .rcl, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 2, .none }, + .{ .rcl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 2, .none }, + .{ .rcl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 2, .long }, + .{ .rcl, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 2, .none }, + .{ .rcl, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 2, .long }, + .{ .rcl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 2, .none }, + .{ .rcl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 2, .long }, - .{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .none }, - .{ .shl, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 4, .rex }, - .{ .shl, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none }, - .{ .shl, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .none }, - .{ .shl, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 4, .long }, - .{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .none }, - .{ .shl, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 4, .rex }, - .{ .shl, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none }, - .{ .shl, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .none }, - .{ .shl, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 4, .long }, - .{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .none }, - .{ .shl, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 4, .rex }, - .{ .shl, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none }, - .{ .shl, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .none }, - .{ .shl, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 4, .long }, + .{ .rcr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 3, .none }, + .{ .rcr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 3, .rex }, + .{ .rcr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 3, .none }, + .{ .rcr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 3, .rex }, + .{ .rcr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 3, .none }, + .{ .rcr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 3, .rex }, + .{ .rcr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 3, .none }, + .{ .rcr, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 3, .none }, + .{ .rcr, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 3, .none }, + .{ .rcr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 3, .none }, + .{ .rcr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 3, .long }, + .{ .rcr, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 3, .none }, + .{ .rcr, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 3, .long }, + .{ .rcr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 3, .none }, + .{ .rcr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 3, .long }, - .{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .none }, - .{ .shr, .m1, .rm8, .unity, .none, .none, 1, 0xd0, 0x00, 0x00, 5, .rex }, - .{ .shr, .m1, .rm16, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none }, - .{ .shr, .m1, .rm32, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .none }, - .{ .shr, .m1, .rm64, .unity, .none, .none, 1, 0xd1, 0x00, 0x00, 5, .long }, - .{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .none }, - .{ .shr, .mc, .rm8, .cl, .none, .none, 1, 0xd2, 0x00, 0x00, 5, .rex }, - .{ .shr, .mc, .rm16, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none }, - .{ .shr, .mc, .rm32, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .none }, - .{ .shr, .mc, .rm64, .cl, .none, .none, 1, 0xd3, 0x00, 0x00, 5, .long }, - .{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .none }, - .{ .shr, .mi, .rm8, .imm8, .none, .none, 1, 0xc0, 0x00, 0x00, 5, .rex }, - .{ .shr, .mi, .rm16, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none }, - .{ .shr, .mi, .rm32, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .none }, - .{ .shr, .mi, .rm64, .imm8, .none, .none, 1, 0xc1, 0x00, 0x00, 5, .long }, + .{ .rol, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 0, .none }, + .{ .rol, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 0, .rex }, + .{ .rol, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 0, .none }, + .{ .rol, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 0, .rex }, + .{ .rol, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 0, .none }, + .{ .rol, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 0, .rex }, + .{ .rol, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 0, .none }, + .{ .rol, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 0, .none }, + .{ .rol, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 0, .none }, + .{ .rol, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 0, .none }, + .{ .rol, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 0, .long }, + .{ .rol, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 0, .none }, + .{ .rol, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 0, .long }, + .{ .rol, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 0, .none }, + .{ .rol, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 0, .long }, - .{ .sub, .zi, .al, .imm8, .none, .none, 1, 0x2c, 0x00, 0x00, 0, .none }, - .{ .sub, .zi, .ax, .imm16, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none }, - .{ .sub, .zi, .eax, .imm32, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .none }, - .{ .sub, .zi, .rax, .imm32s, .none, .none, 1, 0x2d, 0x00, 0x00, 0, .long }, - .{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .none }, - .{ .sub, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 5, .rex }, - .{ .sub, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none }, - .{ .sub, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 5, .none }, - .{ .sub, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 5, .long }, - .{ .sub, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none }, - .{ .sub, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .none }, - .{ .sub, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 5, .long }, - .{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .none }, - .{ .sub, .mr, .rm8, .r8, .none, .none, 1, 0x28, 0x00, 0x00, 0, .rex }, - .{ .sub, .mr, .rm16, .r16, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none }, - .{ .sub, .mr, .rm32, .r32, .none, .none, 1, 0x29, 0x00, 0x00, 0, .none }, - .{ .sub, .mr, .rm64, .r64, .none, .none, 1, 0x29, 0x00, 0x00, 0, .long }, - .{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .none }, - .{ .sub, .rm, .r8, .rm8, .none, .none, 1, 0x2a, 0x00, 0x00, 0, .rex }, - .{ .sub, .rm, .r16, .rm16, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none }, - .{ .sub, .rm, .r32, .rm32, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .none }, - .{ .sub, .rm, .r64, .rm64, .none, .none, 1, 0x2b, 0x00, 0x00, 0, .long }, + .{ .ror, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 1, .none }, + .{ .ror, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 1, .rex }, + .{ .ror, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 1, .none }, + .{ .ror, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 1, .rex }, + .{ .ror, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 1, .none }, + .{ .ror, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 1, .rex }, + .{ .ror, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 1, .none }, + .{ .ror, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 1, .none }, + .{ .ror, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 1, .none }, + .{ .ror, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 1, .none }, + .{ .ror, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 1, .long }, + .{ .ror, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 1, .none }, + .{ .ror, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 1, .long }, + .{ .ror, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 1, .none }, + .{ .ror, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 1, .long }, - .{ .syscall, .np, .none, .none, .none, .none, 2, 0x0f, 0x05, 0x00, 0, .none }, + .{ .sal, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .none }, + .{ .sal, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .rex }, + .{ .sal, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none }, + .{ .sal, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 4, .none }, + .{ .sal, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 4, .long }, + .{ .sal, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .none }, + .{ .sal, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .rex }, + .{ .sal, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 4, .none }, + .{ .sal, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 4, .none }, + .{ .sal, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 4, .long }, + .{ .sal, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .none }, + .{ .sal, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .rex }, + .{ .sal, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 4, .none }, + .{ .sal, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none }, + .{ .sal, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long }, - .{ .@"test", .zi, .al, .imm8, .none, .none, 1, 0xa8, 0x00, 0x00, 0, .none }, - .{ .@"test", .zi, .ax, .imm16, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none }, - .{ .@"test", .zi, .eax, .imm32, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .none }, - .{ .@"test", .zi, .rax, .imm32s, .none, .none, 1, 0xa9, 0x00, 0x00, 0, .long }, - .{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .none }, - .{ .@"test", .mi, .rm8, .imm8, .none, .none, 1, 0xf6, 0x00, 0x00, 0, .rex }, - .{ .@"test", .mi, .rm16, .imm16, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none }, - .{ .@"test", .mi, .rm32, .imm32, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .none }, - .{ .@"test", .mi, .rm64, .imm32s, .none, .none, 1, 0xf7, 0x00, 0x00, 0, .long }, - .{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .none }, - .{ .@"test", .mr, .rm8, .r8, .none, .none, 1, 0x84, 0x00, 0x00, 0, .rex }, - .{ .@"test", .mr, .rm16, .r16, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none }, - .{ .@"test", .mr, .rm32, .r32, .none, .none, 1, 0x85, 0x00, 0x00, 0, .none }, - .{ .@"test", .mr, .rm64, .r64, .none, .none, 1, 0x85, 0x00, 0x00, 0, .long }, + .{ .sar, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 7, .none }, + .{ .sar, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 7, .rex }, + .{ .sar, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 7, .none }, + .{ .sar, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 7, .none }, + .{ .sar, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 7, .long }, + .{ .sar, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 7, .none }, + .{ .sar, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 7, .rex }, + .{ .sar, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 7, .none }, + .{ .sar, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 7, .none }, + .{ .sar, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 7, .long }, + .{ .sar, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 7, .none }, + .{ .sar, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 7, .rex }, + .{ .sar, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 7, .none }, + .{ .sar, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 7, .none }, + .{ .sar, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 7, .long }, - .{ .ud2, .np, .none, .none, .none, .none, 2, 0x0f, 0x0b, 0x00, 0, .none }, + .{ .sbb, .zi, .al, .imm8, .none, .none, &.{ 0x1c }, 0, .none }, + .{ .sbb, .zi, .ax, .imm16, .none, .none, &.{ 0x1d }, 0, .none }, + .{ .sbb, .zi, .eax, .imm32, .none, .none, &.{ 0x1d }, 0, .none }, + .{ .sbb, .zi, .rax, .imm32s, .none, .none, &.{ 0x1d }, 0, .long }, + .{ .sbb, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 3, .none }, + .{ .sbb, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 3, .rex }, + .{ .sbb, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 3, .none }, + .{ .sbb, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 3, .none }, + .{ .sbb, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 3, .long }, + .{ .sbb, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 3, .none }, + .{ .sbb, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 3, .none }, + .{ .sbb, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 3, .long }, + .{ .sbb, .mr, .rm8, .r8, .none, .none, &.{ 0x18 }, 0, .none }, + .{ .sbb, .mr, .rm8, .r8, .none, .none, &.{ 0x18 }, 0, .rex }, + .{ .sbb, .mr, .rm16, .r16, .none, .none, &.{ 0x19 }, 0, .none }, + .{ .sbb, .mr, .rm32, .r32, .none, .none, &.{ 0x19 }, 0, .none }, + .{ .sbb, .mr, .rm64, .r64, .none, .none, &.{ 0x19 }, 0, .long }, + .{ .sbb, .rm, .r8, .rm8, .none, .none, &.{ 0x1a }, 0, .none }, + .{ .sbb, .rm, .r8, .rm8, .none, .none, &.{ 0x1a }, 0, .rex }, + .{ .sbb, .rm, .r16, .rm16, .none, .none, &.{ 0x1b }, 0, .none }, + .{ .sbb, .rm, .r32, .rm32, .none, .none, &.{ 0x1b }, 0, .none }, + .{ .sbb, .rm, .r64, .rm64, .none, .none, &.{ 0x1b }, 0, .long }, - .{ .xor, .zi, .al, .imm8, .none, .none, 1, 0x34, 0x00, 0x00, 0, .none }, - .{ .xor, .zi, .ax, .imm16, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none }, - .{ .xor, .zi, .eax, .imm32, .none, .none, 1, 0x35, 0x00, 0x00, 0, .none }, - .{ .xor, .zi, .rax, .imm32s, .none, .none, 1, 0x35, 0x00, 0x00, 0, .long }, - .{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .none }, - .{ .xor, .mi, .rm8, .imm8, .none, .none, 1, 0x80, 0x00, 0x00, 6, .rex }, - .{ .xor, .mi, .rm16, .imm16, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none }, - .{ .xor, .mi, .rm32, .imm32, .none, .none, 1, 0x81, 0x00, 0x00, 6, .none }, - .{ .xor, .mi, .rm64, .imm32s, .none, .none, 1, 0x81, 0x00, 0x00, 6, .long }, - .{ .xor, .mi, .rm16, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none }, - .{ .xor, .mi, .rm32, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .none }, - .{ .xor, .mi, .rm64, .imm8s, .none, .none, 1, 0x83, 0x00, 0x00, 6, .long }, - .{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .none }, - .{ .xor, .mr, .rm8, .r8, .none, .none, 1, 0x30, 0x00, 0x00, 0, .rex }, - .{ .xor, .mr, .rm16, .r16, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none }, - .{ .xor, .mr, .rm32, .r32, .none, .none, 1, 0x31, 0x00, 0x00, 0, .none }, - .{ .xor, .mr, .rm64, .r64, .none, .none, 1, 0x31, 0x00, 0x00, 0, .long }, - .{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .none }, - .{ .xor, .rm, .r8, .rm8, .none, .none, 1, 0x32, 0x00, 0x00, 0, .rex }, - .{ .xor, .rm, .r16, .rm16, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none }, - .{ .xor, .rm, .r32, .rm32, .none, .none, 1, 0x33, 0x00, 0x00, 0, .none }, - .{ .xor, .rm, .r64, .rm64, .none, .none, 1, 0x33, 0x00, 0x00, 0, .long }, + .{ .scas, .np, .m8, .none, .none, .none, &.{ 0xae }, 0, .none }, + .{ .scas, .np, .m16, .none, .none, .none, &.{ 0xaf }, 0, .none }, + .{ .scas, .np, .m32, .none, .none, .none, &.{ 0xaf }, 0, .none }, + .{ .scas, .np, .m64, .none, .none, .none, &.{ 0xaf }, 0, .long }, + .{ .scasb, .np, .none, .none, .none, .none, &.{ 0xae }, 0, .none }, + .{ .scasw, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .short }, + .{ .scasd, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .none }, + .{ .scasq, .np, .none, .none, .none, .none, &.{ 0xaf }, 0, .long }, + + .{ .seta, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none }, + .{ .seta, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex }, + .{ .setae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none }, + .{ .setae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex }, + .{ .setb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none }, + .{ .setb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex }, + .{ .setbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .none }, + .{ .setbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .rex }, + .{ .setc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none }, + .{ .setc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex }, + .{ .sete, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none }, + .{ .sete, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex }, + .{ .setg, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .none }, + .{ .setg, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .rex }, + .{ .setge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .none }, + .{ .setge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .rex }, + .{ .setl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .none }, + .{ .setl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .rex }, + .{ .setle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .none }, + .{ .setle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .rex }, + .{ .setna, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .none }, + .{ .setna, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x96 }, 0, .rex }, + .{ .setnae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .none }, + .{ .setnae, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x92 }, 0, .rex }, + .{ .setnb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none }, + .{ .setnb, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex }, + .{ .setnbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .none }, + .{ .setnbe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x97 }, 0, .rex }, + .{ .setnc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .none }, + .{ .setnc, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x93 }, 0, .rex }, + .{ .setne, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .none }, + .{ .setne, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .rex }, + .{ .setng, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .none }, + .{ .setng, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9e }, 0, .rex }, + .{ .setnge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .none }, + .{ .setnge, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9c }, 0, .rex }, + .{ .setnl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .none }, + .{ .setnl, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9d }, 0, .rex }, + .{ .setnle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .none }, + .{ .setnle, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9f }, 0, .rex }, + .{ .setno, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x91 }, 0, .none }, + .{ .setno, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x91 }, 0, .rex }, + .{ .setnp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .none }, + .{ .setnp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .rex }, + .{ .setns, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x99 }, 0, .none }, + .{ .setns, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x99 }, 0, .rex }, + .{ .setnz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .none }, + .{ .setnz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x95 }, 0, .rex }, + .{ .seto, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x90 }, 0, .none }, + .{ .seto, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x90 }, 0, .rex }, + .{ .setp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .none }, + .{ .setp, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .rex }, + .{ .setpe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .none }, + .{ .setpe, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9a }, 0, .rex }, + .{ .setpo, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .none }, + .{ .setpo, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x9b }, 0, .rex }, + .{ .sets, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x98 }, 0, .none }, + .{ .sets, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x98 }, 0, .rex }, + .{ .setz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .none }, + .{ .setz, .m, .rm8, .none, .none, .none, &.{ 0x0f, 0x94 }, 0, .rex }, + + .{ .sfence, .np, .none, .none, .none, .none, &.{ 0x0f, 0xae, 0xf8 }, 0, .none }, + + .{ .shl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .none }, + .{ .shl, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 4, .rex }, + .{ .shl, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 4, .none }, + .{ .shl, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 4, .none }, + .{ .shl, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 4, .long }, + .{ .shl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .none }, + .{ .shl, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 4, .rex }, + .{ .shl, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 4, .none }, + .{ .shl, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 4, .none }, + .{ .shl, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 4, .long }, + .{ .shl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .none }, + .{ .shl, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 4, .rex }, + .{ .shl, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 4, .none }, + .{ .shl, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 4, .none }, + .{ .shl, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 4, .long }, + + .{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .none }, + .{ .shr, .m1, .rm8, .unity, .none, .none, &.{ 0xd0 }, 5, .rex }, + .{ .shr, .m1, .rm16, .unity, .none, .none, &.{ 0xd1 }, 5, .none }, + .{ .shr, .m1, .rm32, .unity, .none, .none, &.{ 0xd1 }, 5, .none }, + .{ .shr, .m1, .rm64, .unity, .none, .none, &.{ 0xd1 }, 5, .long }, + .{ .shr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 5, .none }, + .{ .shr, .mc, .rm8, .cl, .none, .none, &.{ 0xd2 }, 5, .rex }, + .{ .shr, .mc, .rm16, .cl, .none, .none, &.{ 0xd3 }, 5, .none }, + .{ .shr, .mc, .rm32, .cl, .none, .none, &.{ 0xd3 }, 5, .none }, + .{ .shr, .mc, .rm64, .cl, .none, .none, &.{ 0xd3 }, 5, .long }, + .{ .shr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 5, .none }, + .{ .shr, .mi, .rm8, .imm8, .none, .none, &.{ 0xc0 }, 5, .rex }, + .{ .shr, .mi, .rm16, .imm8, .none, .none, &.{ 0xc1 }, 5, .none }, + .{ .shr, .mi, .rm32, .imm8, .none, .none, &.{ 0xc1 }, 5, .none }, + .{ .shr, .mi, .rm64, .imm8, .none, .none, &.{ 0xc1 }, 5, .long }, + + .{ .stos, .np, .m8, .none, .none, .none, &.{ 0xaa }, 0, .none }, + .{ .stos, .np, .m16, .none, .none, .none, &.{ 0xab }, 0, .none }, + .{ .stos, .np, .m32, .none, .none, .none, &.{ 0xab }, 0, .none }, + .{ .stos, .np, .m64, .none, .none, .none, &.{ 0xab }, 0, .long }, + .{ .stosb, .np, .none, .none, .none, .none, &.{ 0xaa }, 0, .none }, + .{ .stosw, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .short }, + .{ .stosd, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .none }, + .{ .stosq, .np, .none, .none, .none, .none, &.{ 0xab }, 0, .long }, + + .{ .sub, .zi, .al, .imm8, .none, .none, &.{ 0x2c }, 0, .none }, + .{ .sub, .zi, .ax, .imm16, .none, .none, &.{ 0x2d }, 0, .none }, + .{ .sub, .zi, .eax, .imm32, .none, .none, &.{ 0x2d }, 0, .none }, + .{ .sub, .zi, .rax, .imm32s, .none, .none, &.{ 0x2d }, 0, .long }, + .{ .sub, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 5, .none }, + .{ .sub, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 5, .rex }, + .{ .sub, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 5, .none }, + .{ .sub, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 5, .none }, + .{ .sub, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 5, .long }, + .{ .sub, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 5, .none }, + .{ .sub, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 5, .none }, + .{ .sub, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 5, .long }, + .{ .sub, .mr, .rm8, .r8, .none, .none, &.{ 0x28 }, 0, .none }, + .{ .sub, .mr, .rm8, .r8, .none, .none, &.{ 0x28 }, 0, .rex }, + .{ .sub, .mr, .rm16, .r16, .none, .none, &.{ 0x29 }, 0, .none }, + .{ .sub, .mr, .rm32, .r32, .none, .none, &.{ 0x29 }, 0, .none }, + .{ .sub, .mr, .rm64, .r64, .none, .none, &.{ 0x29 }, 0, .long }, + .{ .sub, .rm, .r8, .rm8, .none, .none, &.{ 0x2a }, 0, .none }, + .{ .sub, .rm, .r8, .rm8, .none, .none, &.{ 0x2a }, 0, .rex }, + .{ .sub, .rm, .r16, .rm16, .none, .none, &.{ 0x2b }, 0, .none }, + .{ .sub, .rm, .r32, .rm32, .none, .none, &.{ 0x2b }, 0, .none }, + .{ .sub, .rm, .r64, .rm64, .none, .none, &.{ 0x2b }, 0, .long }, + + .{ .syscall, .np, .none, .none, .none, .none, &.{ 0x0f, 0x05 }, 0, .none } +, + .{ .@"test", .zi, .al, .imm8, .none, .none, &.{ 0xa8 }, 0, .none }, + .{ .@"test", .zi, .ax, .imm16, .none, .none, &.{ 0xa9 }, 0, .none }, + .{ .@"test", .zi, .eax, .imm32, .none, .none, &.{ 0xa9 }, 0, .none }, + .{ .@"test", .zi, .rax, .imm32s, .none, .none, &.{ 0xa9 }, 0, .long }, + .{ .@"test", .mi, .rm8, .imm8, .none, .none, &.{ 0xf6 }, 0, .none }, + .{ .@"test", .mi, .rm8, .imm8, .none, .none, &.{ 0xf6 }, 0, .rex }, + .{ .@"test", .mi, .rm16, .imm16, .none, .none, &.{ 0xf7 }, 0, .none }, + .{ .@"test", .mi, .rm32, .imm32, .none, .none, &.{ 0xf7 }, 0, .none }, + .{ .@"test", .mi, .rm64, .imm32s, .none, .none, &.{ 0xf7 }, 0, .long }, + .{ .@"test", .mr, .rm8, .r8, .none, .none, &.{ 0x84 }, 0, .none }, + .{ .@"test", .mr, .rm8, .r8, .none, .none, &.{ 0x84 }, 0, .rex }, + .{ .@"test", .mr, .rm16, .r16, .none, .none, &.{ 0x85 }, 0, .none }, + .{ .@"test", .mr, .rm32, .r32, .none, .none, &.{ 0x85 }, 0, .none }, + .{ .@"test", .mr, .rm64, .r64, .none, .none, &.{ 0x85 }, 0, .long }, + + .{ .tzcnt, .rm, .r16, .rm16, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none }, + .{ .tzcnt, .rm, .r32, .rm32, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .none }, + .{ .tzcnt, .rm, .r64, .rm64, .none, .none, &.{ 0xf3, 0x0f, 0xbc }, 0, .long }, + + .{ .ud2, .np, .none, .none, .none, .none, &.{ 0x0f, 0x0b }, 0, .none }, + + .{ .xadd, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xc0 }, 0, .none }, + .{ .xadd, .mr, .rm8, .r8, .none, .none, &.{ 0x0f, 0xc0 }, 0, .rex }, + .{ .xadd, .mr, .rm16, .r16, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none }, + .{ .xadd, .mr, .rm32, .r32, .none, .none, &.{ 0x0f, 0xc1 }, 0, .none }, + .{ .xadd, .mr, .rm64, .r64, .none, .none, &.{ 0x0f, 0xc1 }, 0, .long }, + + .{ .xchg, .o, .ax, .r16, .none, .none, &.{ 0x90 }, 0, .none }, + .{ .xchg, .o, .r16, .ax, .none, .none, &.{ 0x90 }, 0, .none }, + .{ .xchg, .o, .eax, .r32, .none, .none, &.{ 0x90 }, 0, .none }, + .{ .xchg, .o, .rax, .r64, .none, .none, &.{ 0x90 }, 0, .long }, + .{ .xchg, .o, .r32, .eax, .none, .none, &.{ 0x90 }, 0, .none }, + .{ .xchg, .o, .r64, .rax, .none, .none, &.{ 0x90 }, 0, .long }, + .{ .xchg, .mr, .rm8, .r8, .none, .none, &.{ 0x86 }, 0, .none }, + .{ .xchg, .mr, .rm8, .r8, .none, .none, &.{ 0x86 }, 0, .rex }, + .{ .xchg, .rm, .r8, .rm8, .none, .none, &.{ 0x86 }, 0, .none }, + .{ .xchg, .rm, .r8, .rm8, .none, .none, &.{ 0x86 }, 0, .rex }, + .{ .xchg, .mr, .rm16, .r16, .none, .none, &.{ 0x87 }, 0, .none }, + .{ .xchg, .rm, .r16, .rm16, .none, .none, &.{ 0x87 }, 0, .none }, + .{ .xchg, .mr, .rm32, .r32, .none, .none, &.{ 0x87 }, 0, .none }, + .{ .xchg, .mr, .rm64, .r64, .none, .none, &.{ 0x87 }, 0, .long }, + .{ .xchg, .rm, .r32, .rm32, .none, .none, &.{ 0x87 }, 0, .none }, + .{ .xchg, .rm, .r64, .rm64, .none, .none, &.{ 0x87 }, 0, .long }, + + .{ .xor, .zi, .al, .imm8, .none, .none, &.{ 0x34 }, 0, .none }, + .{ .xor, .zi, .ax, .imm16, .none, .none, &.{ 0x35 }, 0, .none }, + .{ .xor, .zi, .eax, .imm32, .none, .none, &.{ 0x35 }, 0, .none }, + .{ .xor, .zi, .rax, .imm32s, .none, .none, &.{ 0x35 }, 0, .long }, + .{ .xor, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 6, .none }, + .{ .xor, .mi, .rm8, .imm8, .none, .none, &.{ 0x80 }, 6, .rex }, + .{ .xor, .mi, .rm16, .imm16, .none, .none, &.{ 0x81 }, 6, .none }, + .{ .xor, .mi, .rm32, .imm32, .none, .none, &.{ 0x81 }, 6, .none }, + .{ .xor, .mi, .rm64, .imm32s, .none, .none, &.{ 0x81 }, 6, .long }, + .{ .xor, .mi, .rm16, .imm8s, .none, .none, &.{ 0x83 }, 6, .none }, + .{ .xor, .mi, .rm32, .imm8s, .none, .none, &.{ 0x83 }, 6, .none }, + .{ .xor, .mi, .rm64, .imm8s, .none, .none, &.{ 0x83 }, 6, .long }, + .{ .xor, .mr, .rm8, .r8, .none, .none, &.{ 0x30 }, 0, .none }, + .{ .xor, .mr, .rm8, .r8, .none, .none, &.{ 0x30 }, 0, .rex }, + .{ .xor, .mr, .rm16, .r16, .none, .none, &.{ 0x31 }, 0, .none }, + .{ .xor, .mr, .rm32, .r32, .none, .none, &.{ 0x31 }, 0, .none }, + .{ .xor, .mr, .rm64, .r64, .none, .none, &.{ 0x31 }, 0, .long }, + .{ .xor, .rm, .r8, .rm8, .none, .none, &.{ 0x32 }, 0, .none }, + .{ .xor, .rm, .r8, .rm8, .none, .none, &.{ 0x32 }, 0, .rex }, + .{ .xor, .rm, .r16, .rm16, .none, .none, &.{ 0x33 }, 0, .none }, + .{ .xor, .rm, .r32, .rm32, .none, .none, &.{ 0x33 }, 0, .none }, + .{ .xor, .rm, .r64, .rm64, .none, .none, &.{ 0x33 }, 0, .long }, // SSE - .{ .addss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x58, 0, .sse }, + .{ .addss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x58 }, 0, .sse }, - .{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, 3, 0xf3, 0x0f, 0xc2, 0, .sse }, + .{ .cmpss, .rmi, .xmm, .xmm_m32, .imm8, .none, &.{ 0xf3, 0x0f, 0xc2 }, 0, .sse }, - .{ .movss, .rm, .xmm, .xmm_m32, .none, .none, 3, 0xf3, 0x0f, 0x10, 0, .sse }, - .{ .movss, .mr, .xmm_m32, .xmm, .none, .none, 3, 0xf3, 0x0f, 0x11, 0, .sse }, + .{ .divss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5e }, 0, .sse }, - .{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, 2, 0x0f, 0x2e, 0x00, 0, .sse }, + .{ .maxss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5f }, 0, .sse }, + + .{ .minss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5d }, 0, .sse }, + + .{ .movss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x10 }, 0, .sse }, + .{ .movss, .mr, .xmm_m32, .xmm, .none, .none, &.{ 0xf3, 0x0f, 0x11 }, 0, .sse }, + + .{ .mulss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x59 }, 0, .sse }, + + .{ .subss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0xf3, 0x0f, 0x5c }, 0, .sse }, + + .{ .ucomiss, .rm, .xmm, .xmm_m32, .none, .none, &.{ 0x0f, 0x2e }, 0, .sse }, // SSE2 - .{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x58, 0, .sse2 }, + .{ .addsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x58 }, 0, .sse2 }, - .{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, 3, 0xf2, 0x0f, 0xc2, 0, .sse2 }, + .{ .cmpsd, .rmi, .xmm, .xmm_m64, .imm8, .none, &.{ 0xf2, 0x0f, 0xc2 }, 0, .sse2 }, - .{ .movq, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf3, 0x0f, 0x7e, 0, .sse2 }, - .{ .movq, .mr, .xmm_m64, .xmm, .none, .none, 3, 0x66, 0x0f, 0xd6, 0, .sse2 }, + .{ .divsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5e }, 0, .sse2 }, - .{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0xf2, 0x0f, 0x10, 0, .sse2 }, - .{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, 3, 0xf2, 0x0f, 0x11, 0, .sse2 }, + .{ .maxsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5f }, 0, .sse2 }, - .{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, 3, 0x66, 0x0f, 0x2e, 0, .sse2 }, + .{ .minsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5d }, 0, .sse2 }, + + .{ .movq, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf3, 0x0f, 0x7e }, 0, .sse2 }, + .{ .movq, .mr, .xmm_m64, .xmm, .none, .none, &.{ 0x66, 0x0f, 0xd6 }, 0, .sse2 }, + + .{ .mulsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x59 }, 0, .sse2 }, + + .{ .subsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x5c }, 0, .sse2 }, + + .{ .movsd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0xf2, 0x0f, 0x10 }, 0, .sse2 }, + .{ .movsd, .mr, .xmm_m64, .xmm, .none, .none, &.{ 0xf2, 0x0f, 0x11 }, 0, .sse2 }, + + .{ .ucomisd, .rm, .xmm, .xmm_m64, .none, .none, &.{ 0x66, 0x0f, 0x2e }, 0, .sse2 }, + + // SSE4.1 + .{ .roundss, .rmi, .xmm, .xmm_m32, .imm8, .none, &.{ 0x66, 0x0f, 0x3a, 0x0a }, 0, .sse4_1 }, + .{ .roundsd, .rmi, .xmm, .xmm_m64, .imm8, .none, &.{ 0x66, 0x0f, 0x3a, 0x0b }, 0, .sse4_1 }, }; // zig fmt: on - diff --git a/src/codegen.zig b/src/codegen.zig index a91795841c..c48200e845 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -608,7 +608,6 @@ pub fn generateSymbol( const payload_type = typed_value.ty.optionalChild(&opt_buf); const is_pl = !typed_value.val.isNull(); const abi_size = math.cast(usize, typed_value.ty.abiSize(target)) orelse return error.Overflow; - const offset = abi_size - (math.cast(usize, payload_type.abiSize(target)) orelse return error.Overflow); if (!payload_type.hasRuntimeBits()) { try code.writer().writeByteNTimes(@boolToInt(is_pl), abi_size); @@ -639,8 +638,8 @@ pub fn generateSymbol( return Result.ok; } + const padding = abi_size - (math.cast(usize, payload_type.abiSize(target)) orelse return error.Overflow) - 1; const value = if (typed_value.val.castTag(.opt_payload)) |payload| payload.data else Value.initTag(.undef); - try code.writer().writeByteNTimes(@boolToInt(is_pl), offset); switch (try generateSymbol(bin_file, src_loc, .{ .ty = payload_type, .val = value, @@ -648,6 +647,8 @@ pub fn generateSymbol( .ok => {}, .fail => |em| return Result{ .fail = em }, } + try code.writer().writeByte(@boolToInt(is_pl)); + try code.writer().writeByteNTimes(0, padding); return Result.ok; }, diff --git a/src/register_manager.zig b/src/register_manager.zig index 4d16348c27..713b669b06 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -305,40 +305,32 @@ pub fn RegisterManager( pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void { const index = indexOfRegIntoTracked(reg) orelse return; log.debug("getReg {} for inst {?}", .{ reg, inst }); - self.markRegAllocated(reg); - if (inst) |tracked_inst| - if (!self.isRegFree(reg)) { - // Move the instruction that was previously there to a - // stack allocation. - const spilled_inst = self.registers[index]; - self.registers[index] = tracked_inst; - try self.getFunction().spillInstruction(reg, spilled_inst); - } else { - self.getRegAssumeFree(reg, tracked_inst); - } - else { - if (!self.isRegFree(reg)) { - // Move the instruction that was previously there to a - // stack allocation. - const spilled_inst = self.registers[index]; - try self.getFunction().spillInstruction(reg, spilled_inst); - self.freeReg(reg); - } - } + if (!self.isRegFree(reg)) { + self.markRegAllocated(reg); + + // Move the instruction that was previously there to a + // stack allocation. + const spilled_inst = self.registers[index]; + if (inst) |tracked_inst| self.registers[index] = tracked_inst; + try self.getFunction().spillInstruction(reg, spilled_inst); + if (inst == null) self.freeReg(reg); + } else self.getRegAssumeFree(reg, inst); } /// Allocates the specified register with the specified /// instruction. Asserts that the register is free and no /// spilling is necessary. - pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void { + pub fn getRegAssumeFree(self: *Self, reg: Register, inst: ?Air.Inst.Index) void { const index = indexOfRegIntoTracked(reg) orelse return; - log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst }); + log.debug("getRegAssumeFree {} for inst {?}", .{ reg, inst }); self.markRegAllocated(reg); assert(self.isRegFree(reg)); - self.registers[index] = inst; - self.markRegUsed(reg); + if (inst) |tracked_inst| { + self.registers[index] = tracked_inst; + self.markRegUsed(reg); + } } /// Marks the specified register as free diff --git a/test/behavior/atomics.zig b/test/behavior/atomics.zig index cf7c3b1503..a1e3af6e9a 100644 --- a/test/behavior/atomics.zig +++ b/test/behavior/atomics.zig @@ -33,7 +33,6 @@ fn testCmpxchg() !void { test "fence" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -44,7 +43,6 @@ test "fence" { test "atomicrmw and atomicload" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -73,7 +71,6 @@ fn testAtomicLoad(ptr: *u8) !void { test "cmpxchg with ptr" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -162,7 +159,6 @@ test "cmpxchg on a global variable" { test "atomic load and rmw with enum" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -180,7 +176,6 @@ test "atomic load and rmw with enum" { test "atomic store" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -194,7 +189,6 @@ test "atomic store" { test "atomic store comptime" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -424,7 +418,6 @@ fn testAtomicsWithType(comptime T: type, a: T, b: T) !void { test "return @atomicStore, using it as a void value" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO diff --git a/test/behavior/bugs/12984.zig b/test/behavior/bugs/12984.zig index fec32947c9..75f2747eda 100644 --- a/test/behavior/bugs/12984.zig +++ b/test/behavior/bugs/12984.zig @@ -14,7 +14,6 @@ pub const CustomDraw = DeleagateWithContext(fn (?OnConfirm) void); test "simple test" { if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO var c: CustomDraw = undefined; _ = c; diff --git a/test/behavior/bugs/13068.zig b/test/behavior/bugs/13068.zig index e28a410807..bfe6164e27 100644 --- a/test/behavior/bugs/13068.zig +++ b/test/behavior/bugs/13068.zig @@ -8,7 +8,6 @@ test { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO list.items.len = 0; diff --git a/test/behavior/bugs/13785.zig b/test/behavior/bugs/13785.zig index d0cced6a79..463cdbec68 100644 --- a/test/behavior/bugs/13785.zig +++ b/test/behavior/bugs/13785.zig @@ -3,7 +3,6 @@ const std = @import("std"); const S = packed struct { a: u0 = 0 }; test { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/call.zig b/test/behavior/call.zig index b51a459932..ab947f69dd 100644 --- a/test/behavior/call.zig +++ b/test/behavior/call.zig @@ -329,7 +329,6 @@ test "inline call preserves tail call" { test "inline call doesn't re-evaluate non generic struct" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO const S = struct { fn foo(f: struct { a: u8, b: u8 }) !void { diff --git a/test/behavior/cast.zig b/test/behavior/cast.zig index 275533d6ec..7d27138ded 100644 --- a/test/behavior/cast.zig +++ b/test/behavior/cast.zig @@ -655,7 +655,6 @@ test "@floatCast cast down" { } test "peer type resolution: unreachable, error set, unreachable" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -1206,7 +1205,6 @@ fn cast128Float(x: u128) f128 { test "implicit cast from *[N]T to ?[*]T" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO var x: ?[*]u16 = null; diff --git a/test/behavior/error.zig b/test/behavior/error.zig index 9d4b154311..a708971a49 100644 --- a/test/behavior/error.zig +++ b/test/behavior/error.zig @@ -451,7 +451,6 @@ test "optional error set is the same size as error set" { } test "nested catch" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -731,7 +730,6 @@ test "pointer to error union payload" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO var err_union: anyerror!u8 = 15; @@ -876,6 +874,7 @@ test "field access of anyerror results in smaller error set" { } test "optional error union return type" { + if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO const S = struct { diff --git a/test/behavior/if.zig b/test/behavior/if.zig index 6632cdd5c2..948629038b 100644 --- a/test/behavior/if.zig +++ b/test/behavior/if.zig @@ -130,7 +130,6 @@ test "if peer expressions inferred optional type" { } test "if-else expression with runtime condition result location is inferred optional" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/maximum_minimum.zig b/test/behavior/maximum_minimum.zig index 34a7d0976a..d538e2db65 100644 --- a/test/behavior/maximum_minimum.zig +++ b/test/behavior/maximum_minimum.zig @@ -5,7 +5,6 @@ const expect = std.testing.expect; const expectEqual = std.testing.expectEqual; test "@max" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -52,7 +51,6 @@ test "@max on vectors" { } test "@min" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/merge_error_sets.zig b/test/behavior/merge_error_sets.zig index 4e6d9e4c45..492cb27699 100644 --- a/test/behavior/merge_error_sets.zig +++ b/test/behavior/merge_error_sets.zig @@ -12,7 +12,6 @@ fn foo() C!void { } test "merge error sets" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (foo()) { diff --git a/test/behavior/null.zig b/test/behavior/null.zig index c78a995833..6ef51cf3bd 100644 --- a/test/behavior/null.zig +++ b/test/behavior/null.zig @@ -29,7 +29,6 @@ test "optional type" { } test "test maybe object and get a pointer to the inner value" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -138,7 +137,6 @@ test "optional pointer to 0 bit type null value at runtime" { } test "if var maybe pointer" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/optional.zig b/test/behavior/optional.zig index bbcc5b3ce6..95b39f2170 100644 --- a/test/behavior/optional.zig +++ b/test/behavior/optional.zig @@ -91,7 +91,6 @@ test "address of unwrap optional" { test "nested optional field in struct" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO const S2 = struct { @@ -109,7 +108,6 @@ test "nested optional field in struct" { test "equality compare optional with non-optional" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO try test_cmp_optional_non_optional(); @@ -227,7 +225,6 @@ test "assigning to an unwrapped optional field in an inline loop" { } test "coerce an anon struct literal to optional struct" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -247,7 +244,6 @@ test "coerce an anon struct literal to optional struct" { } test "0-bit child type coerced to optional return ptr result location" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -299,7 +295,6 @@ test "0-bit child type coerced to optional" { } test "array of optional unaligned types" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -336,7 +331,6 @@ test "array of optional unaligned types" { } test "optional pointer to zero bit optional payload" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -450,7 +444,6 @@ test "Optional slice size is optimized" { test "peer type resolution in nested if expressions" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; const Thing = struct { n: i32 }; var a = false; diff --git a/test/behavior/ptrcast.zig b/test/behavior/ptrcast.zig index 845ea3751e..becdee6b05 100644 --- a/test/behavior/ptrcast.zig +++ b/test/behavior/ptrcast.zig @@ -18,7 +18,6 @@ fn testReinterpretBytesAsInteger() !void { } test "reinterpret an array over multiple elements, with no well-defined layout" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/return_address.zig b/test/behavior/return_address.zig index fe48f21ec2..123b90a66e 100644 --- a/test/behavior/return_address.zig +++ b/test/behavior/return_address.zig @@ -7,7 +7,6 @@ fn retAddr() usize { test "return address" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig index 2a1acebc0f..b250b5b087 100644 --- a/test/behavior/struct.zig +++ b/test/behavior/struct.zig @@ -1149,7 +1149,6 @@ test "anon init through error unions and optionals" { } test "anon init through optional" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -1456,7 +1455,6 @@ test "struct has only one reference" { test "no dependency loop on pointer to optional struct" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO const S = struct { const A = struct { b: B }; @@ -1509,7 +1507,6 @@ test "no dependency loop on optional field wrapped in generic function" { } test "optional field init with tuple" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO diff --git a/test/behavior/switch.zig b/test/behavior/switch.zig index 9129b73f16..1643a2f697 100644 --- a/test/behavior/switch.zig +++ b/test/behavior/switch.zig @@ -228,7 +228,6 @@ const SwitchProngWithVarEnum = union(enum) { }; test "switch prong with variable" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/tuple.zig b/test/behavior/tuple.zig index 79db21424e..3f557bc40e 100644 --- a/test/behavior/tuple.zig +++ b/test/behavior/tuple.zig @@ -263,7 +263,6 @@ test "initializing anon struct with mixed comptime-runtime fields" { test "tuple in tuple passed to generic function" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO const S = struct { @@ -283,7 +282,6 @@ test "tuple in tuple passed to generic function" { test "coerce tuple to tuple" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO const T = std.meta.Tuple(&.{u8}); @@ -298,7 +296,6 @@ test "coerce tuple to tuple" { test "tuple type with void field" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO const T = std.meta.Tuple(&[_]type{void}); const x = T{{}}; @@ -335,7 +332,6 @@ test "zero sized struct in tuple handled correctly" { test "tuple type with void field and a runtime field" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO const T = std.meta.Tuple(&[_]type{ usize, void }); var t: T = .{ 5, {} }; diff --git a/test/behavior/union.zig b/test/behavior/union.zig index 9b49f8bf47..3b040fcba9 100644 --- a/test/behavior/union.zig +++ b/test/behavior/union.zig @@ -1227,7 +1227,6 @@ test "union tag is set when initiated as a temporary value at runtime" { } test "extern union most-aligned field is smaller" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO diff --git a/test/behavior/while.zig b/test/behavior/while.zig index 956aa30f7b..fc3c6e85d8 100644 --- a/test/behavior/while.zig +++ b/test/behavior/while.zig @@ -341,7 +341,6 @@ test "else continue outer while" { } test "try terminating an infinite loop" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO