diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 9629280583..6a87eeb97b 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -275,7 +275,9 @@ pub fn generate( .stack_align = undefined, .end_di_line = module_fn.rbrace_line, .end_di_column = module_fn.rbrace_column, - .mir_to_air_map = if (builtin.mode == .Debug) std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator) else {}, + .mir_to_air_map = if (builtin.mode == .Debug) + std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator) + else {}, }; defer function.stack.deinit(bin_file.allocator); defer function.blocks.deinit(bin_file.allocator); @@ -386,8 +388,8 @@ fn gen(self: *Self) InnerError!void { _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ - .reg1 = .rsp, - .reg2 = .rbp, + .reg1 = .rbp, + .reg2 = .rsp, }).encode(), .data = undefined, }); @@ -1632,18 +1634,18 @@ fn genBinMathOpMir( _ = try self.addInst(.{ .tag = mir_tag, .ops = (Mir.Ops{ - .reg1 = src_reg, - .reg2 = dst_reg, - .flags = 0b11, + .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)), + .reg2 = src_reg, }).encode(), .data = undefined, }); }, .immediate => |imm| { + // TODO I am not quite sure why we need to set the size of the register here... _ = try self.addInst(.{ .tag = mir_tag, .ops = (Mir.Ops{ - .reg1 = dst_reg, + .reg1 = registerAlias(dst_reg, 4), }).encode(), .data = .{ .imm = @intCast(i32, imm) }, }); @@ -1661,7 +1663,7 @@ fn genBinMathOpMir( .tag = mir_tag, .ops = (Mir.Ops{ .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)), - .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)), + .reg2 = .rbp, .flags = 0b01, }).encode(), .data = .{ .imm = -@intCast(i32, adj_off) }, @@ -1691,8 +1693,8 @@ fn genBinMathOpMir( _ = try self.addInst(.{ .tag = mir_tag, .ops = (Mir.Ops{ - .reg1 = registerAlias(src_reg, @intCast(u32, abi_size)), - .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)), + .reg1 = .rbp, + .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)), .flags = 0b10, }).encode(), .data = .{ .imm = -@intCast(i32, adj_off) }, @@ -1741,7 +1743,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) ! _ = try self.addInst(.{ .tag = .imul_complex, .ops = (Mir.Ops{ - .reg1 = dst_reg, + .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)), .reg2 = src_reg, }).encode(), .data = undefined, @@ -1790,7 +1792,7 @@ fn genIMulOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) ! _ = try self.addInst(.{ .tag = .imul_complex, .ops = (Mir.Ops{ - .reg1 = dst_reg, + .reg1 = registerAlias(dst_reg, @divExact(src_reg.size(), 8)), .reg2 = src_reg, }).encode(), .data = undefined, @@ -2868,8 +2870,8 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ - .reg1 = registerAlias(reg, @intCast(u32, abi_size)), - .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)), + .reg1 = .rbp, + .reg2 = registerAlias(reg, @intCast(u32, abi_size)), .flags = 0b10, }).encode(), .data = .{ .imm = -@intCast(i32, adj_off) }, @@ -2926,7 +2928,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void _ = try self.addInst(.{ .tag = tag, .ops = (Mir.Ops{ - .reg1 = reg, + .reg1 = reg.to8(), .flags = flags, }).encode(), .data = undefined, @@ -2952,10 +2954,11 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void } if (x <= math.maxInt(i32)) { // Next best case: if we set the lower four bytes, the upper four will be zeroed. + // TODO I am not quite sure why we need to set the size of the register here... _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ - .reg1 = reg, + .reg1 = registerAlias(reg, 4), }).encode(), .data = .{ .imm = @intCast(i32, x) }, }); @@ -2996,9 +2999,8 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ - .reg1 = reg, + .reg1 = registerAlias(reg, @divExact(src_reg.size(), 8)), .reg2 = src_reg, - .flags = 0b11, }).encode(), .data = undefined, }); @@ -3085,15 +3087,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) { return self.fail("stack offset too large", .{}); } - const ioff = -@intCast(i32, off); _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ .reg1 = registerAlias(reg, @intCast(u32, abi_size)), - .reg2 = registerAlias(.rbp, @intCast(u32, abi_size)), + .reg2 = .rbp, .flags = 0b01, }).encode(), - .data = .{ .imm = ioff }, + .data = .{ .imm = -@intCast(i32, off) }, }); }, } diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index b4d156252e..e410e08ee3 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -11,8 +11,10 @@ const link = @import("../../link.zig"); const log = std.log.scoped(.codegen); const math = std.math; const mem = std.mem; +const testing = std.testing; const Air = @import("../../Air.zig"); +const Allocator = mem.Allocator; const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput; const DW = std.dwarf; const Encoder = bits.Encoder; @@ -72,6 +74,7 @@ pub fn emitMir(emit: *Emit) InnerError!void { .@"or" => try emit.mirArith(.@"or", inst), .sbb => try emit.mirArith(.sbb, inst), .cmp => try emit.mirArith(.cmp, inst), + .mov => try emit.mirArith(.mov, inst), .adc_scale_src => try emit.mirArithScaleSrc(.adc, inst), .add_scale_src => try emit.mirArithScaleSrc(.add, inst), @@ -81,6 +84,7 @@ pub fn emitMir(emit: *Emit) InnerError!void { .or_scale_src => try emit.mirArithScaleSrc(.@"or", inst), .sbb_scale_src => try emit.mirArithScaleSrc(.sbb, inst), .cmp_scale_src => try emit.mirArithScaleSrc(.cmp, inst), + .mov_scale_src => try emit.mirArithScaleSrc(.mov, inst), .adc_scale_dst => try emit.mirArithScaleDst(.adc, inst), .add_scale_dst => try emit.mirArithScaleDst(.add, inst), @@ -90,6 +94,7 @@ pub fn emitMir(emit: *Emit) InnerError!void { .or_scale_dst => try emit.mirArithScaleDst(.@"or", inst), .sbb_scale_dst => try emit.mirArithScaleDst(.sbb, inst), .cmp_scale_dst => try emit.mirArithScaleDst(.cmp, inst), + .mov_scale_dst => try emit.mirArithScaleDst(.mov, inst), .adc_scale_imm => try emit.mirArithScaleImm(.adc, inst), .add_scale_imm => try emit.mirArithScaleImm(.add, inst), @@ -99,14 +104,8 @@ pub fn emitMir(emit: *Emit) InnerError!void { .or_scale_imm => try emit.mirArithScaleImm(.@"or", inst), .sbb_scale_imm => try emit.mirArithScaleImm(.sbb, inst), .cmp_scale_imm => try emit.mirArithScaleImm(.cmp, inst), - - // Even though MOV is technically not an arithmetic op, - // its structure can be represented using the same set of - // opcode primitives. - .mov => try emit.mirArith(.mov, inst), - .mov_scale_src => try emit.mirArithScaleSrc(.mov, inst), - .mov_scale_dst => try emit.mirArithScaleDst(.mov, inst), .mov_scale_imm => try emit.mirArithScaleImm(.mov, inst), + .movabs => try emit.mirMovabs(inst), .lea => try emit.mirLea(inst), @@ -117,16 +116,18 @@ pub fn emitMir(emit: *Emit) InnerError!void { .push => try emit.mirPushPop(.push, inst), .pop => try emit.mirPushPop(.pop, inst), - .jmp => try emit.mirJmpCall(.jmp, inst), - .call => try emit.mirJmpCall(.call, inst), + .jmp => try emit.mirJmpCall(.jmp_near, inst), + .call => try emit.mirJmpCall(.call_near, inst), - .cond_jmp_greater_less => try emit.mirCondJmp(.cond_jmp_greater_less, inst), - .cond_jmp_above_below => try emit.mirCondJmp(.cond_jmp_above_below, inst), - .cond_jmp_eq_ne => try emit.mirCondJmp(.cond_jmp_eq_ne, inst), + .cond_jmp_greater_less, + .cond_jmp_above_below, + .cond_jmp_eq_ne, + => try emit.mirCondJmp(tag, inst), - .cond_set_byte_greater_less => try emit.mirCondSetByte(.cond_set_byte_greater_less, inst), - .cond_set_byte_above_below => try emit.mirCondSetByte(.cond_set_byte_above_below, inst), - .cond_set_byte_eq_ne => try emit.mirCondSetByte(.cond_set_byte_eq_ne, inst), + .cond_set_byte_greater_less, + .cond_set_byte_above_below, + .cond_set_byte_eq_ne, + => try emit.mirCondSetByte(tag, inst), .ret => try emit.mirRet(inst), @@ -184,95 +185,45 @@ fn fixupRelocs(emit: *Emit) InnerError!void { } fn mirBrk(emit: *Emit) InnerError!void { - const encoder = try Encoder.init(emit.code, 1); - encoder.opcode_1byte(0xcc); + return lowerToZoEnc(.brk, emit.code); } fn mirNop(emit: *Emit) InnerError!void { - const encoder = try Encoder.init(emit.code, 1); - encoder.opcode_1byte(0x90); + return lowerToZoEnc(.nop, emit.code); } fn mirSyscall(emit: *Emit) InnerError!void { - const encoder = try Encoder.init(emit.code, 2); - encoder.opcode_2byte(0x0f, 0x05); + return lowerToZoEnc(.syscall, emit.code); } -fn mirPushPop(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { // PUSH/POP reg - const opc: u8 = switch (tag) { - .push => 0x50, - .pop => 0x58, - else => unreachable, - }; - const encoder = try Encoder.init(emit.code, 2); - encoder.rex(.{ - .b = ops.reg1.isExtended(), - }); - encoder.opcode_withReg(opc, ops.reg1.lowId()); + return lowerToOEnc(tag, ops.reg1, emit.code); }, 0b01 => { // PUSH/POP r/m64 const imm = emit.mir.instructions.items(.data)[inst].imm; - const opc: u8 = switch (tag) { - .push => 0xff, - .pop => 0x8f, - else => unreachable, - }; - const modrm_ext: u3 = switch (tag) { - .push => 0x6, - .pop => 0x0, - else => unreachable, - }; - const encoder = try Encoder.init(emit.code, 6); - encoder.opcode_1byte(opc); - if (math.cast(i8, imm)) |imm_i8| { - encoder.modRm_indirectDisp8(modrm_ext, ops.reg1.lowId()); - encoder.imm8(@intCast(i8, imm_i8)); - } else |_| { - encoder.modRm_indirectDisp32(modrm_ext, ops.reg1.lowId()); - encoder.imm32(imm); - } + return lowerToMEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), emit.code); }, 0b10 => { // PUSH imm32 assert(tag == .push); const imm = emit.mir.instructions.items(.data)[inst].imm; - const opc: u8 = if (imm <= math.maxInt(i8)) 0x6a else 0x6b; - const encoder = try Encoder.init(emit.code, 2); - encoder.opcode_1byte(opc); - if (imm <= math.maxInt(i8)) { - encoder.imm8(@intCast(i8, imm)); - } else if (imm <= math.maxInt(i16)) { - encoder.imm16(@intCast(i16, imm)); - } else { - encoder.imm32(imm); - } + return lowerToIEnc(.push, imm, emit.code); }, 0b11 => unreachable, } } -fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const callee_preserved_regs = bits.callee_preserved_regs; - // PUSH/POP reg - const opc: u8 = switch (tag) { - .push => 0x50, - .pop => 0x58, - else => unreachable, - }; - const regs = emit.mir.instructions.items(.data)[inst].regs_to_push_or_pop; if (tag == .push) { for (callee_preserved_regs) |reg, i| { if ((regs >> @intCast(u5, i)) & 1 == 0) continue; - const encoder = try Encoder.init(emit.code, 2); - encoder.rex(.{ - .b = reg.isExtended(), - }); - encoder.opcode_withReg(opc, reg.lowId()); + try lowerToOEnc(.push, reg, emit.code); } } else { // pop in the reverse direction @@ -280,201 +231,89 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Mir.Inst.Tag, inst: M while (i > 0) : (i -= 1) { const reg = callee_preserved_regs[i - 1]; if ((regs >> @intCast(u5, i - 1)) & 1 == 0) continue; - const encoder = try Encoder.init(emit.code, 2); - encoder.rex(.{ - .b = reg.isExtended(), - }); - encoder.opcode_withReg(opc, reg.lowId()); + try lowerToOEnc(.pop, reg, emit.code); } } } -fn mirJmpCall(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const flag = @truncate(u1, ops.flags); if (flag == 0) { const target = emit.mir.instructions.items(.data)[inst].inst; - const opc: u8 = switch (tag) { - .jmp => 0xe9, - .call => 0xe8, - else => unreachable, - }; const source = emit.code.items.len; - const encoder = try Encoder.init(emit.code, 5); - encoder.opcode_1byte(opc); + try lowerToDEnc(tag, 0, emit.code); try emit.relocs.append(emit.bin_file.allocator, .{ .source = source, .target = target, - .offset = emit.code.items.len, + .offset = emit.code.items.len - 4, .length = 5, }); - encoder.imm32(0x0); return; } - const modrm_ext: u3 = switch (tag) { - .jmp => 0x4, - .call => 0x2, - else => unreachable, - }; if (ops.reg1 == .none) { // JMP/CALL [imm] const imm = emit.mir.instructions.items(.data)[inst].imm; - const encoder = try Encoder.init(emit.code, 7); - encoder.opcode_1byte(0xff); - encoder.modRm_SIBDisp0(modrm_ext); - encoder.sib_disp32(); - encoder.imm32(imm); - return; + return lowerToMEnc(tag, RegisterOrMemory.mem(null, imm), emit.code); } // JMP/CALL reg - const encoder = try Encoder.init(emit.code, 2); - encoder.opcode_1byte(0xff); - encoder.modRm_direct(modrm_ext, ops.reg1.lowId()); + return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); } -const CondType = enum { - /// greater than or equal - gte, - - /// greater than - gt, - - /// less than - lt, - - /// less than or equal - lte, - - /// above or equal - ae, - - /// above - a, - - /// below - b, - - /// below or equal - be, - - /// not equal - ne, - - /// equal - eq, - - fn fromTagAndFlags(tag: Mir.Inst.Tag, flags: u2) CondType { - return switch (tag) { - .cond_jmp_greater_less, - .cond_set_byte_greater_less, - => switch (flags) { - 0b00 => CondType.gte, - 0b01 => CondType.gt, - 0b10 => CondType.lt, - 0b11 => CondType.lte, - }, - .cond_jmp_above_below, - .cond_set_byte_above_below, - => switch (flags) { - 0b00 => CondType.ae, - 0b01 => CondType.a, - 0b10 => CondType.b, - 0b11 => CondType.be, - }, - .cond_jmp_eq_ne, - .cond_set_byte_eq_ne, - => switch (@truncate(u1, flags)) { - 0b0 => CondType.ne, - 0b1 => CondType.eq, - }, - else => unreachable, - }; - } -}; - -inline fn getCondOpCode(tag: Mir.Inst.Tag, cond: CondType) u8 { - switch (cond) { - .gte => return switch (tag) { - .cond_jmp_greater_less => 0x8d, - .cond_set_byte_greater_less => 0x9d, - else => unreachable, - }, - .gt => return switch (tag) { - .cond_jmp_greater_less => 0x8f, - .cond_set_byte_greater_less => 0x9f, - else => unreachable, - }, - .lt => return switch (tag) { - .cond_jmp_greater_less => 0x8c, - .cond_set_byte_greater_less => 0x9c, - else => unreachable, - }, - .lte => return switch (tag) { - .cond_jmp_greater_less => 0x8e, - .cond_set_byte_greater_less => 0x9e, - else => unreachable, - }, - .ae => return switch (tag) { - .cond_jmp_above_below => 0x83, - .cond_set_byte_above_below => 0x93, - else => unreachable, - }, - .a => return switch (tag) { - .cond_jmp_above_below => 0x87, - .cond_set_byte_greater_less => 0x97, - else => unreachable, - }, - .b => return switch (tag) { - .cond_jmp_above_below => 0x82, - .cond_set_byte_greater_less => 0x92, - else => unreachable, - }, - .be => return switch (tag) { - .cond_jmp_above_below => 0x86, - .cond_set_byte_greater_less => 0x96, - else => unreachable, - }, - .eq => return switch (tag) { - .cond_jmp_eq_ne => 0x84, - .cond_set_byte_eq_ne => 0x94, - else => unreachable, - }, - .ne => return switch (tag) { - .cond_jmp_eq_ne => 0x85, - .cond_set_byte_eq_ne => 0x95, - else => unreachable, - }, - } -} - -fn mirCondJmp(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const target = emit.mir.instructions.items(.data)[inst].inst; - const cond = CondType.fromTagAndFlags(tag, ops.flags); - const opc = getCondOpCode(tag, cond); + const tag = switch (mir_tag) { + .cond_jmp_greater_less => switch (ops.flags) { + 0b00 => Tag.jge, + 0b01 => Tag.jg, + 0b10 => Tag.jl, + 0b11 => Tag.jle, + }, + .cond_jmp_above_below => switch (ops.flags) { + 0b00 => Tag.jae, + 0b01 => Tag.ja, + 0b10 => Tag.jb, + 0b11 => Tag.jbe, + }, + .cond_jmp_eq_ne => switch (@truncate(u1, ops.flags)) { + 0b0 => Tag.jne, + 0b1 => Tag.je, + }, + else => unreachable, + }; const source = emit.code.items.len; - const encoder = try Encoder.init(emit.code, 6); - encoder.opcode_2byte(0x0f, opc); + try lowerToDEnc(tag, 0, emit.code); try emit.relocs.append(emit.bin_file.allocator, .{ .source = source, .target = target, - .offset = emit.code.items.len, + .offset = emit.code.items.len - 4, .length = 6, }); - encoder.imm32(0); } -fn mirCondSetByte(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); - const cond = CondType.fromTagAndFlags(tag, ops.flags); - const opc = getCondOpCode(tag, cond); - const encoder = try Encoder.init(emit.code, 4); - encoder.rex(.{ - .w = true, - .b = ops.reg1.isExtended(), - }); - encoder.opcode_2byte(0x0f, opc); - encoder.modRm_direct(0x0, ops.reg1.lowId()); + const tag = switch (mir_tag) { + .cond_set_byte_greater_less => switch (ops.flags) { + 0b00 => Tag.setge, + 0b01 => Tag.setg, + 0b10 => Tag.setl, + 0b11 => Tag.setle, + }, + .cond_set_byte_above_below => switch (ops.flags) { + 0b00 => Tag.setae, + 0b01 => Tag.seta, + 0b10 => Tag.setb, + 0b11 => Tag.setbe, + }, + .cond_set_byte_eq_ne => switch (@truncate(u1, ops.flags)) { + 0b0 => Tag.setne, + 0b1 => Tag.sete, + }, + else => unreachable, + }; + return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); } fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { @@ -482,31 +321,17 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { assert(tag == .@"test"); const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { - 0b00 => blk: { + 0b00 => { if (ops.reg2 == .none) { // TEST r/m64, imm32 + // MI const imm = emit.mir.instructions.items(.data)[inst].imm; if (ops.reg1.to64() == .rax) { - // TODO reduce the size of the instruction if the immediate - // is smaller than 32 bits - const encoder = try Encoder.init(emit.code, 6); - encoder.rex(.{ - .w = true, - }); - encoder.opcode_1byte(0xa9); - encoder.imm32(imm); - break :blk; + // TEST rax, imm32 + // I + return lowerToIEnc(.@"test", imm, emit.code); } - const opc: u8 = if (ops.reg1.size() == 8) 0xf6 else 0xf7; - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = true, - .b = ops.reg1.isExtended(), - }); - encoder.opcode_1byte(opc); - encoder.modRm_direct(0, ops.reg1.lowId()); - encoder.imm8(@intCast(i8, imm)); - break :blk; + return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code); } // TEST r/m64, r64 return emit.fail("TODO TEST r/m64, r64", .{}); @@ -519,26 +344,161 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .ret); const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); - const encoder = try Encoder.init(emit.code, 3); switch (ops.flags) { 0b00 => { // RETF imm16 + // I const imm = emit.mir.instructions.items(.data)[inst].imm; - encoder.opcode_1byte(0xca); - encoder.imm16(@intCast(i16, imm)); + return lowerToIEnc(.ret_far, imm, emit.code); }, - 0b01 => encoder.opcode_1byte(0xcb), // RETF + 0b01 => return lowerToZoEnc(.ret_far, emit.code), 0b10 => { // RET imm16 + // I const imm = emit.mir.instructions.items(.data)[inst].imm; - encoder.opcode_1byte(0xc2); - encoder.imm16(@intCast(i16, imm)); + return lowerToIEnc(.ret_near, imm, emit.code); }, - 0b11 => encoder.opcode_1byte(0xc3), // RET + 0b11 => return lowerToZoEnc(.ret_near, emit.code), } } -const EncType = enum { +const Tag = enum { + adc, + add, + sub, + xor, + @"and", + @"or", + sbb, + cmp, + mov, + lea, + jmp_near, + call_near, + push, + pop, + @"test", + brk, + nop, + imul, + syscall, + ret_near, + ret_far, + jo, + jno, + jb, + jbe, + jc, + jnae, + jnc, + jae, + je, + jz, + jne, + jnz, + jna, + jnb, + jnbe, + ja, + js, + jns, + jpe, + jp, + jpo, + jnp, + jnge, + jl, + jge, + jnl, + jle, + jng, + jg, + jnle, + seto, + setno, + setb, + setc, + setnae, + setnb, + setnc, + setae, + sete, + setz, + setne, + setnz, + setbe, + setna, + seta, + setnbe, + sets, + setns, + setp, + setpe, + setnp, + setop, + setl, + setnge, + setnl, + setge, + setle, + setng, + setnle, + setg, + + fn isSetCC(tag: Tag) bool { + return switch (tag) { + .seto, + .setno, + .setb, + .setc, + .setnae, + .setnb, + .setnc, + .setae, + .sete, + .setz, + .setne, + .setnz, + .setbe, + .setna, + .seta, + .setnbe, + .sets, + .setns, + .setp, + .setpe, + .setnp, + .setop, + .setl, + .setnge, + .setnl, + .setge, + .setle, + .setng, + .setnle, + .setg, + => true, + else => false, + }; + } +}; + +const Encoding = enum { + /// OP + zo, + + /// OP rel32 + d, + + /// OP r/m64 + m, + + /// OP r64 + o, + + /// OP imm32 + i, + /// OP r/m64, imm32 mi, @@ -547,223 +507,853 @@ const EncType = enum { /// OP r64, r/m64 rm, + + /// OP r64, imm64 + oi, + + /// OP al/ax/eax/rax, moffs + fd, + + /// OP moffs, al/ax/eax/rax + td, + + /// OP r64, r/m64, imm32 + rmi, }; -const OpCode = struct { - opc: u8, - /// Only used if `EncType == .mi`. - modrm_ext: u3, +const OpCode = union(enum) { + one_byte: u8, + two_byte: struct { _1: u8, _2: u8 }, + + fn oneByte(opc: u8) OpCode { + return .{ .one_byte = opc }; + } + + fn twoByte(opc1: u8, opc2: u8) OpCode { + return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } }; + } + + fn encode(opc: OpCode, encoder: Encoder) void { + switch (opc) { + .one_byte => |v| encoder.opcode_1byte(v), + .two_byte => |v| encoder.opcode_2byte(v._1, v._2), + } + } + + fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void { + assert(opc == .one_byte); + encoder.opcode_withReg(opc.one_byte, reg.lowId()); + } }; -inline fn getArithOpCode(tag: Mir.Inst.Tag, enc: EncType) OpCode { +inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { switch (enc) { + .zo => return switch (tag) { + .ret_near => OpCode.oneByte(0xc3), + .ret_far => OpCode.oneByte(0xcb), + .brk => OpCode.oneByte(0xcc), + .nop => OpCode.oneByte(0x90), + .syscall => OpCode.twoByte(0x0f, 0x05), + else => null, + }, + .d => return switch (tag) { + .jmp_near => OpCode.oneByte(0xe9), + .call_near => OpCode.oneByte(0xe8), + .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80), + .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81), + .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82), + .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83), + .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84), + .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85), + .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86), + .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87), + .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88), + .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89), + .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a), + .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b), + .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c), + .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d), + .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e), + .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f), + else => null, + }, + .m => return switch (tag) { + .jmp_near, .call_near, .push => OpCode.oneByte(0xff), + .pop => OpCode.oneByte(0x8f), + .seto => OpCode.twoByte(0x0f, 0x90), + .setno => OpCode.twoByte(0x0f, 0x91), + .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92), + .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93), + .sete, .setz => OpCode.twoByte(0x0f, 0x94), + .setne, .setnz => OpCode.twoByte(0x0f, 0x95), + .setbe, .setna => OpCode.twoByte(0x0f, 0x96), + .seta, .setnbe => OpCode.twoByte(0x0f, 0x97), + .sets => OpCode.twoByte(0x0f, 0x98), + .setns => OpCode.twoByte(0x0f, 0x99), + .setp, .setpe => OpCode.twoByte(0x0f, 0x9a), + .setnp, .setop => OpCode.twoByte(0x0f, 0x9b), + .setl, .setnge => OpCode.twoByte(0x0f, 0x9c), + .setnl, .setge => OpCode.twoByte(0x0f, 0x9d), + .setle, .setng => OpCode.twoByte(0x0f, 0x9e), + .setnle, .setg => OpCode.twoByte(0x0f, 0x9f), + else => null, + }, + .o => return switch (tag) { + .push => OpCode.oneByte(0x50), + .pop => OpCode.oneByte(0x58), + else => null, + }, + .i => return switch (tag) { + .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68), + .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9), + .ret_near => OpCode.oneByte(0xc2), + .ret_far => OpCode.oneByte(0xca), + else => null, + }, .mi => return switch (tag) { - .adc => .{ .opc = 0x81, .modrm_ext = 0x2 }, - .add => .{ .opc = 0x81, .modrm_ext = 0x0 }, - .sub => .{ .opc = 0x81, .modrm_ext = 0x5 }, - .xor => .{ .opc = 0x81, .modrm_ext = 0x6 }, - .@"and" => .{ .opc = 0x81, .modrm_ext = 0x4 }, - .@"or" => .{ .opc = 0x81, .modrm_ext = 0x1 }, - .sbb => .{ .opc = 0x81, .modrm_ext = 0x3 }, - .cmp => .{ .opc = 0x81, .modrm_ext = 0x7 }, - .mov => .{ .opc = 0xc7, .modrm_ext = 0x0 }, - else => unreachable, + .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81), + .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7), + .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7), + else => null, }, - .mr => { - const opc: u8 = switch (tag) { - .adc => 0x11, - .add => 0x01, - .sub => 0x29, - .xor => 0x31, - .@"and" => 0x21, - .@"or" => 0x09, - .sbb => 0x19, - .cmp => 0x39, - .mov => 0x89, - else => unreachable, - }; - return .{ .opc = opc, .modrm_ext = undefined }; + .mr => return switch (tag) { + .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11), + .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01), + .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29), + .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31), + .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21), + .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09), + .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19), + .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39), + .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89), + else => null, }, - .rm => { - const opc: u8 = switch (tag) { - .adc => 0x13, - .add => 0x03, - .sub => 0x2b, - .xor => 0x33, - .@"and" => 0x23, - .@"or" => 0x0b, - .sbb => 0x1b, - .cmp => 0x3b, - .mov => 0x8b, - else => unreachable, - }; - return .{ .opc = opc, .modrm_ext = undefined }; + .rm => return switch (tag) { + .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13), + .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03), + .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b), + .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33), + .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23), + .@"or" => OpCode.oneByte(if (is_one_byte) 0x0b else 0x0b), + .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b), + .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b), + .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b), + .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d), + .imul => OpCode.twoByte(0x0f, 0xaf), + else => null, + }, + .oi => return switch (tag) { + .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8), + else => null, + }, + .fd => return switch (tag) { + .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1), + else => null, + }, + .td => return switch (tag) { + .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3), + else => null, + }, + .rmi => return switch (tag) { + .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69), + else => null, }, } } -fn mirArith(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); - switch (ops.flags) { - 0b00 => blk: { - if (ops.reg2 == .none) { - // OP reg1, imm32 - // OP r/m64, imm32 - const imm = emit.mir.instructions.items(.data)[inst].imm; - const opcode = getArithOpCode(tag, .mi); - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .b = ops.reg1.isExtended(), - }); - if (tag != .mov and imm <= math.maxInt(i8)) { - encoder.opcode_1byte(opcode.opc + 2); - encoder.modRm_direct(opcode.modrm_ext, ops.reg1.lowId()); - encoder.imm8(@intCast(i8, imm)); - } else { - encoder.opcode_1byte(opcode.opc); - encoder.modRm_direct(opcode.modrm_ext, ops.reg1.lowId()); - encoder.imm32(imm); +inline fn getModRmExt(tag: Tag) ?u3 { + return switch (tag) { + .adc => 0x2, + .add => 0x0, + .sub => 0x5, + .xor => 0x6, + .@"and" => 0x4, + .@"or" => 0x1, + .sbb => 0x3, + .cmp => 0x7, + .mov => 0x0, + .jmp_near => 0x4, + .call_near => 0x2, + .push => 0x6, + .pop => 0x0, + .@"test" => 0x0, + .seto, + .setno, + .setb, + .setc, + .setnae, + .setnb, + .setnc, + .setae, + .sete, + .setz, + .setne, + .setnz, + .setbe, + .setna, + .seta, + .setnbe, + .sets, + .setns, + .setp, + .setpe, + .setnp, + .setop, + .setl, + .setnge, + .setnl, + .setge, + .setle, + .setng, + .setnle, + .setg, + => 0x0, + else => null, + }; +} + +const ScaleIndexBase = struct { + scale: u2, + index_reg: ?Register, + base_reg: ?Register, +}; + +const Memory = struct { + reg: ?Register, + rip: bool = false, + disp: i32, + sib: ?ScaleIndexBase = null, +}; + +const RegisterOrMemory = union(enum) { + register: Register, + memory: Memory, + + fn reg(register: Register) RegisterOrMemory { + return .{ .register = register }; + } + + fn mem(register: ?Register, disp: i32) RegisterOrMemory { + return .{ + .memory = .{ + .reg = register, + .disp = disp, + }, + }; + } + + fn rip(disp: i32) RegisterOrMemory { + return .{ + .memory = .{ + .reg = null, + .rip = true, + .disp = disp, + }, + }; + } +}; + +fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void { + const opc = getOpCode(tag, .zo, false).?; + const encoder = try Encoder.init(code, 1); + opc.encode(encoder); +} + +fn lowerToIEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void { + if (tag == .ret_far or tag == .ret_near) { + const encoder = try Encoder.init(code, 3); + const opc = getOpCode(tag, .i, false).?; + opc.encode(encoder); + encoder.imm16(@intCast(i16, imm)); + return; + } + const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?; + const encoder = try Encoder.init(code, 5); + if (immOpSize(imm) == 16) { + encoder.opcode_1byte(0x66); + } + opc.encode(encoder); + if (immOpSize(imm) == 8) { + encoder.imm8(@intCast(i8, imm)); + } else if (immOpSize(imm) == 16) { + encoder.imm16(@intCast(i16, imm)); + } else { + encoder.imm32(imm); + } +} + +fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void { + if (reg.size() != 16 and reg.size() != 64) return error.EmitFail; // TODO correct for push/pop, but is it universal? + const opc = getOpCode(tag, .o, false).?; + const encoder = try Encoder.init(code, 3); + if (reg.size() == 16) { + encoder.opcode_1byte(0x66); + } + encoder.rex(.{ + .w = false, + .b = reg.isExtended(), + }); + opc.encodeWithReg(encoder, reg); +} + +fn lowerToDEnc(tag: Tag, imm: i32, code: *std.ArrayList(u8)) InnerError!void { + const opc = getOpCode(tag, .d, false).?; + const encoder = try Encoder.init(code, 6); + opc.encode(encoder); + encoder.imm32(imm); +} + +fn lowerToMEnc(tag: Tag, reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8)) InnerError!void { + const opc = getOpCode(tag, .m, false).?; + const modrm_ext = getModRmExt(tag).?; + switch (reg_or_mem) { + .register => |reg| { + // TODO clean this up! + if (reg.size() != 64) { + if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail; + } + const encoder = try Encoder.init(code, 3); + encoder.rex(.{ + .w = tag.isSetCC(), + .b = reg.isExtended(), + }); + opc.encode(encoder); + encoder.modRm_direct(modrm_ext, reg.lowId()); + }, + .memory => |mem_op| { + const encoder = try Encoder.init(code, 8); + if (mem_op.reg) |reg| { + // TODO clean this up! + if (reg.size() != 64) { + if (reg.size() != 8 and !tag.isSetCC()) return error.EmitFail; } - break :blk; - } - // OP reg1, reg2 - // OP r/m64, r64 - const opcode = getArithOpCode(tag, .mr); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; - const encoder = try Encoder.init(emit.code, 3); - encoder.rex(.{ - .w = ops.reg1.size() == 64 and ops.reg2.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(opc); - encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId()); - }, - 0b01 => blk: { - const imm = emit.mir.instructions.items(.data)[inst].imm; - const opcode = getArithOpCode(tag, .rm); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; - if (ops.reg2 == .none) { - // OP reg1, [imm32] - // OP r64, r/m64 - const encoder = try Encoder.init(emit.code, 8); encoder.rex(.{ - .w = ops.reg1.size() == 64, - .b = ops.reg1.isExtended(), + .w = tag.isSetCC(), + .b = reg.isExtended(), }); - encoder.opcode_1byte(opc); - encoder.modRm_SIBDisp0(ops.reg1.lowId()); - encoder.sib_disp32(); - encoder.disp32(imm); - break :blk; - } - // OP reg1, [reg2 + imm32] - // OP r64, r/m64 - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(opc); - if (imm <= math.maxInt(i8)) { - encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId()); - encoder.disp8(@intCast(i8, imm)); - } else { - encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId()); - encoder.disp32(imm); - } - }, - 0b10 => blk: { - if (ops.reg2 == .none) { - // OP [reg1 + 0], imm32 - // OP r/m64, imm32 - const imm = emit.mir.instructions.items(.data)[inst].imm; - const opcode = getArithOpCode(tag, .mi); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .b = ops.reg1.isExtended(), - }); - encoder.opcode_1byte(opc); - encoder.modRm_indirectDisp0(opcode.modrm_ext, ops.reg1.lowId()); - if (imm <= math.maxInt(i8)) { - encoder.imm8(@intCast(i8, imm)); - } else if (imm <= math.maxInt(i16)) { - encoder.imm16(@intCast(i16, imm)); + opc.encode(encoder); + if (reg.lowId() == 4) { + if (mem_op.disp == 0) { + encoder.modRm_SIBDisp0(modrm_ext); + encoder.sib_base(reg.lowId()); + } else if (immOpSize(mem_op.disp) == 8) { + encoder.modRm_SIBDisp8(modrm_ext); + encoder.sib_baseDisp8(reg.lowId()); + encoder.disp8(@intCast(i8, mem_op.disp)); + } else { + encoder.modRm_SIBDisp32(modrm_ext); + encoder.sib_baseDisp32(reg.lowId()); + encoder.disp32(mem_op.disp); + } } else { - encoder.imm32(imm); + if (mem_op.disp == 0) { + encoder.modRm_indirectDisp0(modrm_ext, reg.lowId()); + } else if (immOpSize(mem_op.disp) == 8) { + encoder.modRm_indirectDisp8(modrm_ext, reg.lowId()); + encoder.disp8(@intCast(i8, mem_op.disp)); + } else { + encoder.modRm_indirectDisp32(modrm_ext, reg.lowId()); + encoder.disp32(mem_op.disp); + } } - break :blk; - } - // OP [reg1 + imm32], reg2 - // OP r/m64, r64 - const imm = emit.mir.instructions.items(.data)[inst].imm; - const opcode = getArithOpCode(tag, .mr); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg2.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(opc); - if (imm <= math.maxInt(i8)) { - encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId()); - encoder.disp8(@intCast(i8, imm)); } else { - encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId()); - encoder.disp32(imm); + opc.encode(encoder); + if (mem_op.rip) { + encoder.modRm_RIPDisp32(modrm_ext); + } else { + encoder.modRm_SIBDisp0(modrm_ext); + encoder.sib_disp32(); + } + encoder.disp32(mem_op.disp); } }, - 0b11 => blk: { - if (ops.reg2 == .none) { - // OP [reg1 + imm32], imm32 - // OP r/m64, imm32 - const payload = emit.mir.instructions.items(.data)[inst].payload; - const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; - const opcode = getArithOpCode(tag, .mi); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; - const encoder = try Encoder.init(emit.code, 11); + } +} + +fn lowerToTdEnc(tag: Tag, moffs: i64, reg: Register, code: *std.ArrayList(u8)) InnerError!void { + return lowerToTdFdEnc(tag, reg, moffs, code, true); +} + +fn lowerToFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8)) InnerError!void { + return lowerToTdFdEnc(tag, reg, moffs, code, false); +} + +fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: i64, code: *std.ArrayList(u8), td: bool) InnerError!void { + if (reg.lowId() != Register.rax.lowId()) return error.EmitFail; + if (reg.size() != immOpSize(moffs)) return error.EmitFail; + const opc = if (td) + getOpCode(tag, .td, reg.size() == 8).? + else + getOpCode(tag, .fd, reg.size() == 8).?; + const encoder = try Encoder.init(code, 10); + if (reg.size() == 16) { + encoder.opcode_1byte(0x66); + } + encoder.rex(.{ + .w = reg.size() == 64, + }); + opc.encode(encoder); + switch (reg.size()) { + 8 => { + const moffs8 = try math.cast(i8, moffs); + encoder.imm8(moffs8); + }, + 16 => { + const moffs16 = try math.cast(i16, moffs); + encoder.imm16(moffs16); + }, + 32 => { + const moffs32 = try math.cast(i32, moffs); + encoder.imm32(moffs32); + }, + 64 => { + encoder.imm64(@bitCast(u64, moffs)); + }, + else => unreachable, + } +} + +fn lowerToOiEnc(tag: Tag, reg: Register, imm: i64, code: *std.ArrayList(u8)) InnerError!void { + if (reg.size() != immOpSize(imm)) return error.EmitFail; + const opc = getOpCode(tag, .oi, reg.size() == 8).?; + const encoder = try Encoder.init(code, 10); + if (reg.size() == 16) { + encoder.opcode_1byte(0x66); + } + encoder.rex(.{ + .w = reg.size() == 64, + .b = reg.isExtended(), + }); + opc.encodeWithReg(encoder, reg); + switch (reg.size()) { + 8 => { + const imm8 = try math.cast(i8, imm); + encoder.imm8(imm8); + }, + 16 => { + const imm16 = try math.cast(i16, imm); + encoder.imm16(imm16); + }, + 32 => { + const imm32 = try math.cast(i32, imm); + encoder.imm32(imm32); + }, + 64 => { + encoder.imm64(@bitCast(u64, imm)); + }, + else => unreachable, + } +} + +fn lowerToMiEnc(tag: Tag, reg_or_mem: RegisterOrMemory, imm: i32, code: *std.ArrayList(u8)) InnerError!void { + const modrm_ext = getModRmExt(tag).?; + switch (reg_or_mem) { + .register => |dst_reg| { + const opc = getOpCode(tag, .mi, dst_reg.size() == 8).?; + const encoder = try Encoder.init(code, 7); + if (dst_reg.size() == 16) { + // 0x66 prefix switches to the non-default size; here we assume a switch from + // the default 32bits to 16bits operand-size. + // More info: https://www.cs.uni-potsdam.de/desn/lehre/ss15/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf#page=32&zoom=auto,-159,773 + encoder.opcode_1byte(0x66); + } + encoder.rex(.{ + .w = dst_reg.size() == 64, + .b = dst_reg.isExtended(), + }); + opc.encode(encoder); + encoder.modRm_direct(modrm_ext, dst_reg.lowId()); + switch (dst_reg.size()) { + 8 => { + const imm8 = try math.cast(i8, imm); + encoder.imm8(imm8); + }, + 16 => { + const imm16 = try math.cast(i16, imm); + encoder.imm16(imm16); + }, + 32, 64 => encoder.imm32(imm), + else => unreachable, + } + }, + .memory => |dst_mem| { + const opc = getOpCode(tag, .mi, false).?; + const encoder = try Encoder.init(code, 12); + if (dst_mem.reg) |dst_reg| { + // Register dst_reg can either be 64bit or 32bit in size. + // TODO for memory operand, immediate operand pair, we currently + // have no way of flagging whether the immediate can be 8-, 16- or + // 32-bit and whether the corresponding memory operand is respectively + // a byte, word or dword ptr. + // TODO we currently don't have a way to flag imm32 64bit sign extended + if (dst_reg.size() != 64) return error.EmitFail; encoder.rex(.{ .w = false, - .b = ops.reg1.isExtended(), + .b = dst_reg.isExtended(), }); - encoder.opcode_1byte(opc); - if (imm_pair.dest_off <= math.maxInt(i8)) { - encoder.modRm_indirectDisp8(opcode.modrm_ext, ops.reg1.lowId()); - encoder.disp8(@intCast(i8, imm_pair.dest_off)); + opc.encode(encoder); + if (dst_reg.lowId() == 4) { + if (dst_mem.disp == 0) { + encoder.modRm_SIBDisp0(modrm_ext); + encoder.sib_base(dst_reg.lowId()); + } else if (immOpSize(dst_mem.disp) == 8) { + encoder.modRm_SIBDisp8(modrm_ext); + encoder.sib_baseDisp8(dst_reg.lowId()); + encoder.disp8(@intCast(i8, dst_mem.disp)); + } else { + encoder.modRm_SIBDisp32(modrm_ext); + encoder.sib_baseDisp32(dst_reg.lowId()); + encoder.disp32(dst_mem.disp); + } } else { - encoder.modRm_indirectDisp32(opcode.modrm_ext, ops.reg1.lowId()); - encoder.disp32(imm_pair.dest_off); + if (dst_mem.disp == 0) { + encoder.modRm_indirectDisp0(modrm_ext, dst_reg.lowId()); + } else if (immOpSize(dst_mem.disp) == 8) { + encoder.modRm_indirectDisp8(modrm_ext, dst_reg.lowId()); + encoder.disp8(@intCast(i8, dst_mem.disp)); + } else { + encoder.modRm_indirectDisp32(modrm_ext, dst_reg.lowId()); + encoder.disp32(dst_mem.disp); + } } - encoder.imm32(imm_pair.operand); - break :blk; + } else { + opc.encode(encoder); + if (dst_mem.rip) { + encoder.modRm_RIPDisp32(modrm_ext); + } else { + encoder.modRm_SIBDisp0(modrm_ext); + encoder.sib_disp32(); + } + encoder.disp32(dst_mem.disp); } - // TODO clearly mov doesn't belong here; for other, arithemtic ops, - // this is the same as 0b00. - const opcode = getArithOpCode(tag, if (tag == .mov) .rm else .mr); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; - const encoder = try Encoder.init(emit.code, 3); - encoder.rex(.{ - .w = ops.reg1.size() == 64 and ops.reg2.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(opc); - encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId()); + encoder.imm32(imm); }, } } -fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +fn lowerToRmEnc( + tag: Tag, + reg: Register, + reg_or_mem: RegisterOrMemory, + code: *std.ArrayList(u8), +) InnerError!void { + const opc = getOpCode(tag, .rm, reg.size() == 8).?; + switch (reg_or_mem) { + .register => |src_reg| { + if (reg.size() != src_reg.size()) return error.EmitFail; + const encoder = try Encoder.init(code, 3); + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + opc.encode(encoder); + encoder.modRm_direct(reg.lowId(), src_reg.lowId()); + }, + .memory => |src_mem| { + const encoder = try Encoder.init(code, 9); + if (reg.size() == 16) { + encoder.opcode_1byte(0x66); + } + if (src_mem.reg) |src_reg| { + // TODO handle 32-bit base register - requires prefix 0x67 + // Intel Manual, Vol 1, chapter 3.6 and 3.6.1 + if (src_reg.size() != 64) return error.EmitFail; + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + opc.encode(encoder); + if (src_reg.lowId() == 4) { + if (src_mem.disp == 0) { + encoder.modRm_SIBDisp0(reg.lowId()); + encoder.sib_base(src_reg.lowId()); + } else if (immOpSize(src_mem.disp) == 8) { + encoder.modRm_SIBDisp8(reg.lowId()); + encoder.sib_baseDisp8(src_reg.lowId()); + encoder.disp8(@intCast(i8, src_mem.disp)); + } else { + encoder.modRm_SIBDisp32(reg.lowId()); + encoder.sib_baseDisp32(src_reg.lowId()); + encoder.disp32(src_mem.disp); + } + } else { + if (src_mem.disp == 0) { + encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId()); + } else if (immOpSize(src_mem.disp) == 8) { + encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId()); + encoder.disp8(@intCast(i8, src_mem.disp)); + } else { + encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId()); + encoder.disp32(src_mem.disp); + } + } + } else { + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + }); + opc.encode(encoder); + if (src_mem.rip) { + encoder.modRm_RIPDisp32(reg.lowId()); + } else { + encoder.modRm_SIBDisp0(reg.lowId()); + encoder.sib_disp32(); + } + encoder.disp32(src_mem.disp); + } + }, + } +} + +fn lowerToMrEnc( + tag: Tag, + reg_or_mem: RegisterOrMemory, + reg: Register, + code: *std.ArrayList(u8), +) InnerError!void { + // We use size of source register reg to work out which + // variant of memory ptr to pick: + // * reg is 64bit - qword ptr + // * reg is 32bit - dword ptr + // * reg is 16bit - word ptr + // * reg is 8bit - byte ptr + const opc = getOpCode(tag, .mr, reg.size() == 8).?; + switch (reg_or_mem) { + .register => |dst_reg| { + if (dst_reg.size() != reg.size()) return error.EmitFail; + const encoder = try Encoder.init(code, 3); + encoder.rex(.{ + .w = dst_reg.size() == 64, + .r = reg.isExtended(), + .b = dst_reg.isExtended(), + }); + opc.encode(encoder); + encoder.modRm_direct(reg.lowId(), dst_reg.lowId()); + }, + .memory => |dst_mem| { + const encoder = try Encoder.init(code, 9); + if (reg.size() == 16) { + encoder.opcode_1byte(0x66); + } + if (dst_mem.reg) |dst_reg| { + if (dst_reg.size() != 64) return error.EmitFail; + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + .b = dst_reg.isExtended(), + }); + opc.encode(encoder); + if (dst_reg.lowId() == 4) { + if (dst_mem.disp == 0) { + encoder.modRm_SIBDisp0(reg.lowId()); + encoder.sib_base(dst_reg.lowId()); + } else if (immOpSize(dst_mem.disp) == 8) { + encoder.modRm_SIBDisp8(reg.lowId()); + encoder.sib_baseDisp8(dst_reg.lowId()); + encoder.disp8(@intCast(i8, dst_mem.disp)); + } else { + encoder.modRm_SIBDisp32(reg.lowId()); + encoder.sib_baseDisp32(dst_reg.lowId()); + encoder.disp32(dst_mem.disp); + } + } else { + if (dst_mem.disp == 0) { + encoder.modRm_indirectDisp0(reg.lowId(), dst_reg.lowId()); + } else if (immOpSize(dst_mem.disp) == 8) { + encoder.modRm_indirectDisp8(reg.lowId(), dst_reg.lowId()); + encoder.disp8(@intCast(i8, dst_mem.disp)); + } else { + encoder.modRm_indirectDisp32(reg.lowId(), dst_reg.lowId()); + encoder.disp32(dst_mem.disp); + } + } + } else { + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + }); + opc.encode(encoder); + if (dst_mem.rip) { + encoder.modRm_RIPDisp32(reg.lowId()); + } else { + encoder.modRm_SIBDisp0(reg.lowId()); + encoder.sib_disp32(); + } + encoder.disp32(dst_mem.disp); + } + }, + } +} + +fn lowerToRmiEnc( + tag: Tag, + reg: Register, + reg_or_mem: RegisterOrMemory, + imm: i32, + code: *std.ArrayList(u8), +) InnerError!void { + const opc = getOpCode(tag, .rmi, reg.size() == 8).?; + switch (reg_or_mem) { + .register => |src_reg| { + if (reg.size() != src_reg.size()) return error.EmitFail; + const encoder = try Encoder.init(code, 7); + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + opc.encode(encoder); + encoder.modRm_direct(reg.lowId(), src_reg.lowId()); + switch (reg.size()) { + 8 => { + const imm8 = try math.cast(i8, imm); + encoder.imm8(imm8); + }, + 16 => { + const imm16 = try math.cast(i16, imm); + encoder.imm16(imm16); + }, + 32, 64 => encoder.imm32(imm), + else => unreachable, + } + }, + .memory => |src_mem| { + const encoder = try Encoder.init(code, 13); + if (reg.size() == 16) { + encoder.opcode_1byte(0x66); + } + if (src_mem.reg) |src_reg| { + // TODO handle 32-bit base register - requires prefix 0x67 + // Intel Manual, Vol 1, chapter 3.6 and 3.6.1 + if (src_reg.size() != 64) return error.EmitFail; + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + opc.encode(encoder); + if (src_reg.lowId() == 4) { + if (src_mem.disp == 0) { + encoder.modRm_SIBDisp0(reg.lowId()); + encoder.sib_base(src_reg.lowId()); + } else if (immOpSize(src_mem.disp) == 8) { + encoder.modRm_SIBDisp8(reg.lowId()); + encoder.sib_baseDisp8(src_reg.lowId()); + encoder.disp8(@intCast(i8, src_mem.disp)); + } else { + encoder.modRm_SIBDisp32(reg.lowId()); + encoder.sib_baseDisp32(src_reg.lowId()); + encoder.disp32(src_mem.disp); + } + } else { + if (src_mem.disp == 0) { + encoder.modRm_indirectDisp0(reg.lowId(), src_reg.lowId()); + } else if (immOpSize(src_mem.disp) == 8) { + encoder.modRm_indirectDisp8(reg.lowId(), src_reg.lowId()); + encoder.disp8(@intCast(i8, src_mem.disp)); + } else { + encoder.modRm_indirectDisp32(reg.lowId(), src_reg.lowId()); + encoder.disp32(src_mem.disp); + } + } + } else { + encoder.rex(.{ + .w = reg.size() == 64, + .r = reg.isExtended(), + }); + opc.encode(encoder); + if (src_mem.rip) { + encoder.modRm_RIPDisp32(reg.lowId()); + } else { + encoder.modRm_SIBDisp0(reg.lowId()); + encoder.sib_disp32(); + } + encoder.disp32(src_mem.disp); + } + encoder.imm32(imm); + }, + } +} + +fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { + const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + switch (ops.flags) { + 0b00 => { + if (ops.reg2 == .none) { + // mov reg1, imm32 + // MI + const imm = emit.mir.instructions.items(.data)[inst].imm; + return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code); + } + // mov reg1, reg2 + // RM + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + }, + 0b01 => { + const imm = emit.mir.instructions.items(.data)[inst].imm; + if (ops.reg2 == .none) { + // mov reg1, [imm32] + // RM + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(null, imm), emit.code); + } + // mov reg1, [reg2 + imm32] + // RM + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code); + }, + 0b10 => { + if (ops.reg2 == .none) { + // mov dword ptr [reg1 + 0], imm32 + // MI + const imm = emit.mir.instructions.items(.data)[inst].imm; + return lowerToMiEnc(tag, RegisterOrMemory.mem(ops.reg1, 0), imm, emit.code); + } + // mov [reg1 + imm32], reg2 + // MR + const imm = emit.mir.instructions.items(.data)[inst].imm; + return lowerToMrEnc(tag, RegisterOrMemory.mem(ops.reg1, imm), ops.reg2, emit.code); + }, + 0b11 => { + if (ops.reg2 == .none) { + // mov dword ptr [reg1 + imm32], imm32 + // MI + const payload = emit.mir.instructions.items(.data)[inst].payload; + const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; + return lowerToMiEnc( + tag, + RegisterOrMemory.mem(ops.reg1, imm_pair.dest_off), + imm_pair.operand, + emit.code, + ); + } + return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{}); + }, + } +} + +fn immOpSize(imm: i64) u8 { + blk: { + _ = math.cast(i8, imm) catch break :blk; + return 8; + } + blk: { + _ = math.cast(i16, imm) catch break :blk; + return 16; + } + blk: { + _ = math.cast(i32, imm) catch break :blk; + return 32; + } + return 64; +} + +// TODO +fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const scale = ops.flags; // OP reg1, [reg2 + scale*rcx + imm32] - const opcode = getArithOpCode(tag, .rm); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; + const opc = getOpCode(tag, .rm, ops.reg1.size() == 8).?; const imm = emit.mir.instructions.items(.data)[inst].imm; const encoder = try Encoder.init(emit.code, 8); encoder.rex(.{ @@ -771,7 +1361,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE .r = ops.reg1.isExtended(), .b = ops.reg2.isExtended(), }); - encoder.opcode_1byte(opc); + opc.encode(encoder); if (imm <= math.maxInt(i8)) { encoder.modRm_SIBDisp8(ops.reg1.lowId()); encoder.sib_scaleIndexBaseDisp8(scale, Register.rcx.lowId(), ops.reg2.lowId()); @@ -783,22 +1373,23 @@ fn mirArithScaleSrc(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE } } -fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +// TODO +fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const scale = ops.flags; const imm = emit.mir.instructions.items(.data)[inst].imm; if (ops.reg2 == .none) { // OP [reg1 + scale*rax + 0], imm32 - const opcode = getArithOpCode(tag, .mi); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; + const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?; + const modrm_ext = getModRmExt(tag).?; const encoder = try Encoder.init(emit.code, 8); encoder.rex(.{ .w = ops.reg1.size() == 64, .b = ops.reg1.isExtended(), }); - encoder.opcode_1byte(opc); - encoder.modRm_SIBDisp0(opcode.modrm_ext); + opc.encode(encoder); + encoder.modRm_SIBDisp0(modrm_ext); encoder.sib_scaleIndexBase(scale, Register.rax.lowId(), ops.reg1.lowId()); if (imm <= math.maxInt(i8)) { encoder.imm8(@intCast(i8, imm)); @@ -811,15 +1402,14 @@ fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE } // OP [reg1 + scale*rax + imm32], reg2 - const opcode = getArithOpCode(tag, .mr); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; + const opc = getOpCode(tag, .mr, ops.reg1.size() == 8).?; const encoder = try Encoder.init(emit.code, 8); encoder.rex(.{ .w = ops.reg1.size() == 64, .r = ops.reg2.isExtended(), .b = ops.reg1.isExtended(), }); - encoder.opcode_1byte(opc); + opc.encode(encoder); if (imm <= math.maxInt(i8)) { encoder.modRm_SIBDisp8(ops.reg2.lowId()); encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId()); @@ -831,25 +1421,26 @@ fn mirArithScaleDst(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerE } } -fn mirArithScaleImm(emit: *Emit, tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { +// TODO +fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const scale = ops.flags; const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; - const opcode = getArithOpCode(tag, .mi); - const opc = if (ops.reg1.size() == 8) opcode.opc - 1 else opcode.opc; + const opc = getOpCode(tag, .mi, ops.reg1.size() == 8).?; + const modrm_ext = getModRmExt(tag).?; const encoder = try Encoder.init(emit.code, 2); encoder.rex(.{ .w = ops.reg1.size() == 64, .b = ops.reg1.isExtended(), }); - encoder.opcode_1byte(opc); + opc.encode(encoder); if (imm_pair.dest_off <= math.maxInt(i8)) { - encoder.modRm_SIBDisp8(opcode.modrm_ext); + encoder.modRm_SIBDisp8(modrm_ext); encoder.sib_scaleIndexBaseDisp8(scale, Register.rax.lowId(), ops.reg1.lowId()); encoder.disp8(@intCast(i8, imm_pair.dest_off)); } else { - encoder.modRm_SIBDisp32(opcode.modrm_ext); + encoder.modRm_SIBDisp32(modrm_ext); encoder.sib_scaleIndexBaseDisp32(scale, Register.rax.lowId(), ops.reg1.lowId()); encoder.disp32(imm_pair.dest_off); } @@ -860,54 +1451,24 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .movabs); const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); - - const encoder = try Encoder.init(emit.code, 10); - const is_64 = blk: { - if (ops.flags == 0b00) { - // movabs reg, imm64 - const opc: u8 = if (ops.reg1.size() == 8) 0xb0 else 0xb8; - if (ops.reg1.size() == 64) { - encoder.rex(.{ - .w = true, - .b = ops.reg1.isExtended(), - }); - encoder.opcode_withReg(opc, ops.reg1.lowId()); - break :blk true; - } - break :blk false; - } - if (ops.reg1 == .none) { - // movabs moffs64, rax - const opc: u8 = if (ops.reg2.size() == 8) 0xa2 else 0xa3; - encoder.rex(.{ - .w = ops.reg2.size() == 64, - }); - encoder.opcode_1byte(opc); - break :blk ops.reg2.size() == 64; - } else { - // movabs rax, moffs64 - const opc: u8 = if (ops.reg2.size() == 8) 0xa0 else 0xa1; - encoder.rex(.{ - .w = ops.reg1.size() == 64, - }); - encoder.opcode_1byte(opc); - break :blk ops.reg1.size() == 64; - } - }; - - if (is_64) { + const imm: i64 = if (ops.reg1.size() == 64) blk: { const payload = emit.mir.instructions.items(.data)[inst].payload; - const imm64 = emit.mir.extraData(Mir.Imm64, payload).data; - encoder.imm64(imm64.decode()); + const imm = emit.mir.extraData(Mir.Imm64, payload).data; + break :blk @bitCast(i64, imm.decode()); + } else emit.mir.instructions.items(.data)[inst].imm; + if (ops.flags == 0b00) { + // movabs reg, imm64 + // OI + return lowerToOiEnc(.mov, ops.reg1, imm, emit.code); + } + if (ops.reg1 == .none) { + // movabs moffs64, rax + // TD + return lowerToTdEnc(.mov, imm, ops.reg2, emit.code); } else { - const imm = emit.mir.instructions.items(.data)[inst].imm; - if (imm <= math.maxInt(i8)) { - encoder.imm8(@intCast(i8, imm)); - } else if (imm <= math.maxInt(i16)) { - encoder.imm16(@intCast(i16, imm)); - } else { - encoder.imm32(imm); - } + // movabs rax, moffs64 + // FD + return lowerToFdEnc(.mov, ops.reg1, imm, emit.code); } } @@ -916,34 +1477,10 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { assert(tag == .imul_complex); const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { - 0b00 => { - const encoder = try Encoder.init(emit.code, 4); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_2byte(0x0f, 0xaf); - encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId()); - }, + 0b00 => return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code), 0b10 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - const opc: u8 = if (imm <= math.maxInt(i8)) 0x6b else 0x69; - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg1.isExtended(), - }); - encoder.opcode_1byte(opc); - encoder.modRm_direct(ops.reg1.lowId(), ops.reg2.lowId()); - if (imm <= math.maxInt(i8)) { - encoder.imm8(@intCast(i8, imm)); - } else if (imm <= math.maxInt(i16)) { - encoder.imm16(@intCast(i16, imm)); - } else { - encoder.imm32(imm); - } + return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code); }, else => return emit.fail("TODO implement imul", .{}), } @@ -955,37 +1492,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); assert(ops.flags == 0b01); const imm = emit.mir.instructions.items(.data)[inst].imm; - - if (imm == 0) { - const encoder = try Encoder.init(emit.code, 3); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(0x8d); - encoder.modRm_indirectDisp0(ops.reg1.lowId(), ops.reg2.lowId()); - } else if (imm <= math.maxInt(i8)) { - const encoder = try Encoder.init(emit.code, 4); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(0x8d); - encoder.modRm_indirectDisp8(ops.reg1.lowId(), ops.reg2.lowId()); - encoder.disp8(@intCast(i8, imm)); - } else { - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - .b = ops.reg2.isExtended(), - }); - encoder.opcode_1byte(0x8d); - encoder.modRm_indirectDisp32(ops.reg1.lowId(), ops.reg2.lowId()); - encoder.disp32(imm); - } + return lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.mem(ops.reg2, imm), emit.code); } fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { @@ -993,26 +1500,22 @@ fn mirLeaRip(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { assert(tag == .lea_rip); const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const start_offset = emit.code.items.len; - const encoder = try Encoder.init(emit.code, 7); - encoder.rex(.{ - .w = ops.reg1.size() == 64, - .r = ops.reg1.isExtended(), - }); - encoder.opcode_1byte(0x8d); - encoder.modRm_RIPDisp32(ops.reg1.lowId()); + try lowerToRmEnc(.lea, ops.reg1, RegisterOrMemory.rip(0), emit.code); const end_offset = emit.code.items.len; if (@truncate(u1, ops.flags) == 0b0) { + // Backpatch the displacement + // TODO figure out if this can be simplified const payload = emit.mir.instructions.items(.data)[inst].payload; const imm = emit.mir.extraData(Mir.Imm64, payload).data.decode(); - encoder.disp32(@intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset + 4))); + const disp = @intCast(i32, @intCast(i64, imm) - @intCast(i64, end_offset - start_offset)); + mem.writeIntLittle(i32, emit.code.items[end_offset - 4 ..][0..4], disp); } else { const got_entry = emit.mir.instructions.items(.data)[inst].got_entry; - encoder.disp32(0); if (emit.bin_file.cast(link.File.MachO)) |macho_file| { // TODO I think the reloc might be in the wrong place. const decl = macho_file.active_decl.?; try decl.link.macho.relocs.append(emit.bin_file.allocator, .{ - .offset = @intCast(u32, end_offset), + .offset = @intCast(u32, end_offset - 4), .target = .{ .local = got_entry }, .addend = 0, .subtractor = null, @@ -1031,12 +1534,9 @@ fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { assert(tag == .call_extern); const n_strx = emit.mir.instructions.items(.data)[inst].extern_fn; const offset = blk: { - const offset = @intCast(u32, emit.code.items.len + 1); // callq - const encoder = try Encoder.init(emit.code, 5); - encoder.opcode_1byte(0xe8); - encoder.imm32(0x0); - break :blk offset; + try lowerToDEnc(.call_near, 0, emit.code); + break :blk @intCast(u32, emit.code.items.len) - 4; }; if (emit.bin_file.cast(link.File.MachO)) |macho_file| { // Add relocation to the decl. @@ -1206,3 +1706,230 @@ fn addDbgInfoTypeReloc(emit: *Emit, ty: Type) !void { .none => {}, } } + +fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void { + assert(expected.len > 0); + if (mem.eql(u8, expected, given)) return; + const expected_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(expected)}); + defer testing.allocator.free(expected_fmt); + const given_fmt = try std.fmt.allocPrint(testing.allocator, "{x}", .{std.fmt.fmtSliceHexLower(given)}); + defer testing.allocator.free(given_fmt); + const idx = mem.indexOfDiff(u8, expected_fmt, given_fmt).?; + var padding = try testing.allocator.alloc(u8, idx + 5); + defer testing.allocator.free(padding); + mem.set(u8, padding, ' '); + std.debug.print("\nASM: {s}\nEXP: {s}\nGIV: {s}\n{s}^ -- first differing byte\n", .{ + assembly, + expected_fmt, + given_fmt, + padding, + }); + return error.TestFailed; +} + +const TestEmitCode = struct { + buf: std.ArrayList(u8), + next: usize = 0, + + fn init() TestEmitCode { + return .{ + .buf = std.ArrayList(u8).init(testing.allocator), + }; + } + + fn deinit(emit: *TestEmitCode) void { + emit.buf.deinit(); + emit.next = undefined; + } + + fn buffer(emit: *TestEmitCode) *std.ArrayList(u8) { + emit.next = emit.buf.items.len; + return &emit.buf; + } + + fn emitted(emit: TestEmitCode) []const u8 { + return emit.buf.items[emit.next..]; + } +}; + +test "lower MI encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToMiEnc(.mov, RegisterOrMemory.reg(.rax), 0x10, code.buffer()); + try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", code.emitted(), "mov rax, 0x10"); + try lowerToMiEnc(.mov, RegisterOrMemory.mem(.r11, 0), 0x10, code.buffer()); + try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", code.emitted(), "mov dword ptr [r11 + 0], 0x10"); + try lowerToMiEnc(.add, RegisterOrMemory.mem(.rdx, -8), 0x10, code.buffer()); + try expectEqualHexStrings("\x81\x42\xF8\x10\x00\x00\x00", code.emitted(), "add dword ptr [rdx - 8], 0x10"); + try lowerToMiEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), 0x10, code.buffer()); + try expectEqualHexStrings( + "\x41\x81\xab\x00\x00\x00\x10\x10\x00\x00\x00", + code.emitted(), + "sub dword ptr [r11 + 0x10000000], 0x10", + ); + try lowerToMiEnc(.@"and", RegisterOrMemory.mem(null, 0x10000000), 0x10, code.buffer()); + try expectEqualHexStrings( + "\x81\x24\x25\x00\x00\x00\x10\x10\x00\x00\x00", + code.emitted(), + "and dword ptr [ds:0x10000000], 0x10", + ); + try lowerToMiEnc(.@"and", RegisterOrMemory.mem(.r12, 0x10000000), 0x10, code.buffer()); + try expectEqualHexStrings( + "\x41\x81\xA4\x24\x00\x00\x00\x10\x10\x00\x00\x00", + code.emitted(), + "and dword ptr [r12 + 0x10000000], 0x10", + ); + try lowerToMiEnc(.mov, RegisterOrMemory.rip(0x10), 0x10, code.buffer()); + try expectEqualHexStrings( + "\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00", + code.emitted(), + "mov [rip + 0x10], 0x10", + ); +} + +test "lower RM encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), code.buffer()); + try expectEqualHexStrings("\x48\x8b\xc3", code.emitted(), "mov rax, rbx"); + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.r11, 0), code.buffer()); + try expectEqualHexStrings("\x49\x8b\x03", code.emitted(), "mov rax, qword ptr [r11 + 0]"); + try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(null, 0x10000000), code.buffer()); + try expectEqualHexStrings( + "\x4C\x03\x1C\x25\x00\x00\x00\x10", + code.emitted(), + "add r11, qword ptr [ds:0x10000000]", + ); + try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(null, 0x10000000), code.buffer()); + try expectEqualHexStrings( + "\x44\x02\x24\x25\x00\x00\x00\x10", + code.emitted(), + "add r11b, byte ptr [ds:0x10000000]", + ); + try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r13, 0x10000000), code.buffer()); + try expectEqualHexStrings( + "\x4D\x2B\x9D\x00\x00\x00\x10", + code.emitted(), + "sub r11, qword ptr [r13 + 0x10000000]", + ); + try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.r12, 0x10000000), code.buffer()); + try expectEqualHexStrings( + "\x4D\x2B\x9C\x24\x00\x00\x00\x10", + code.emitted(), + "sub r11, qword ptr [r12 + 0x10000000]", + ); + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.rbp, -4), code.buffer()); + try expectEqualHexStrings("\x48\x8B\x45\xFC", code.emitted(), "mov rax, qword ptr [rbp - 4]"); + try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(0x10), code.buffer()); + try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", code.emitted(), "lea rax, [rip + 0x10]"); +} + +test "lower MR encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, code.buffer()); + try expectEqualHexStrings("\x48\x89\xd8", code.emitted(), "mov rax, rbx"); + try lowerToMrEnc(.mov, RegisterOrMemory.mem(.rbp, -4), .r11, code.buffer()); + try expectEqualHexStrings("\x4c\x89\x5d\xfc", code.emitted(), "mov qword ptr [rbp - 4], r11"); + try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12b, code.buffer()); + try expectEqualHexStrings( + "\x44\x00\x24\x25\x00\x00\x00\x10", + code.emitted(), + "add byte ptr [ds:0x10000000], r12b", + ); + try lowerToMrEnc(.add, RegisterOrMemory.mem(null, 0x10000000), .r12d, code.buffer()); + try expectEqualHexStrings( + "\x44\x01\x24\x25\x00\x00\x00\x10", + code.emitted(), + "add dword ptr [ds:0x10000000], r12d", + ); + try lowerToMrEnc(.sub, RegisterOrMemory.mem(.r11, 0x10000000), .r12, code.buffer()); + try expectEqualHexStrings( + "\x4D\x29\xA3\x00\x00\x00\x10", + code.emitted(), + "sub qword ptr [r11 + 0x10000000], r12", + ); + try lowerToMrEnc(.mov, RegisterOrMemory.rip(0x10), .r12, code.buffer()); + try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", code.emitted(), "mov qword ptr [rip + 0x10], r12"); +} + +test "lower OI encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToOiEnc(.mov, .rax, 0x1000000000000000, code.buffer()); + try expectEqualHexStrings( + "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", + code.emitted(), + "movabs rax, 0x1000000000000000", + ); + try lowerToOiEnc(.mov, .r11, 0x1000000000000000, code.buffer()); + try expectEqualHexStrings( + "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", + code.emitted(), + "movabs r11, 0x1000000000000000", + ); + try lowerToOiEnc(.mov, .r11d, 0x10000000, code.buffer()); + try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", code.emitted(), "mov r11d, 0x10000000"); + try lowerToOiEnc(.mov, .r11w, 0x1000, code.buffer()); + try expectEqualHexStrings("\x66\x41\xBB\x00\x10", code.emitted(), "mov r11w, 0x1000"); + try lowerToOiEnc(.mov, .r11b, 0x10, code.buffer()); + try expectEqualHexStrings("\x41\xB3\x10", code.emitted(), "mov r11b, 0x10"); +} + +test "lower FD/TD encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToFdEnc(.mov, .rax, 0x1000000000000000, code.buffer()); + try expectEqualHexStrings( + "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10", + code.emitted(), + "mov rax, ds:0x1000000000000000", + ); + try lowerToFdEnc(.mov, .eax, 0x10000000, code.buffer()); + try expectEqualHexStrings("\xa1\x00\x00\x00\x10", code.emitted(), "mov eax, ds:0x10000000"); + try lowerToFdEnc(.mov, .ax, 0x1000, code.buffer()); + try expectEqualHexStrings("\x66\xa1\x00\x10", code.emitted(), "mov ax, ds:0x1000"); + try lowerToFdEnc(.mov, .al, 0x10, code.buffer()); + try expectEqualHexStrings("\xa0\x10", code.emitted(), "mov al, ds:0x10"); +} + +test "lower M encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToMEnc(.jmp_near, RegisterOrMemory.reg(.r12), code.buffer()); + try expectEqualHexStrings("\x41\xFF\xE4", code.emitted(), "jmp r12"); + try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0), code.buffer()); + try expectEqualHexStrings("\x41\xFF\x24\x24", code.emitted(), "jmp qword ptr [r12]"); + try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x10), code.buffer()); + try expectEqualHexStrings("\x41\xFF\x64\x24\x10", code.emitted(), "jmp qword ptr [r12 + 0x10]"); + try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(.r12, 0x1000), code.buffer()); + try expectEqualHexStrings( + "\x41\xFF\xA4\x24\x00\x10\x00\x00", + code.emitted(), + "jmp qword ptr [r12 + 0x1000]", + ); + try lowerToMEnc(.jmp_near, RegisterOrMemory.rip(0x10), code.buffer()); + try expectEqualHexStrings("\xFF\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [rip + 0x10]"); + try lowerToMEnc(.jmp_near, RegisterOrMemory.mem(null, 0x10), code.buffer()); + try expectEqualHexStrings("\xFF\x24\x25\x10\x00\x00\x00", code.emitted(), "jmp qword ptr [ds:0x10]"); + try lowerToMEnc(.seta, RegisterOrMemory.reg(.r11b), code.buffer()); + try expectEqualHexStrings("\x49\x0F\x97\xC3", code.emitted(), "seta r11b"); +} + +test "lower O encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToOEnc(.pop, .r12, code.buffer()); + try expectEqualHexStrings("\x41\x5c", code.emitted(), "pop r12"); + try lowerToOEnc(.push, .r12w, code.buffer()); + try expectEqualHexStrings("\x66\x41\x54", code.emitted(), "push r12w"); +} + +test "lower RMI encoding" { + var code = TestEmitCode.init(); + defer code.deinit(); + try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.rbp, -8), 0x10, code.buffer()); + try expectEqualHexStrings("\x48\x69\x45\xF8\x10\x00\x00\x00", code.emitted(), "imul rax, [rbp - 8], 0x10"); + try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, code.buffer()); + try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", code.emitted(), "imul r12, r12, 0x10"); +} diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 501a5428d2..5bd3c53004 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -140,6 +140,7 @@ pub const Inst = struct { mov_scale_src, mov_scale_dst, mov_scale_imm, + lea, lea_scale_src, lea_scale_dst,