diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 6a073158e6..2a392f378b 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -37,6 +37,7 @@ const abi = @import("abi.zig"); const Lower = @import("Lower.zig"); const Register = bits.Register; +const CSR = bits.CSR; const Immediate = bits.Immediate; const Memory = bits.Memory; const FrameIndex = bits.FrameIndex; @@ -87,6 +88,9 @@ exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .{}, /// across each runtime branch upon joining. branch_stack: *std.ArrayList(Branch), +// The current bit length of vector registers. +vec_len: u32, + // Key is the block instruction blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{}, register_manager: RegisterManager = .{}, @@ -747,6 +751,7 @@ pub fn generate( .end_di_line = func.rbrace_line, .end_di_column = func.rbrace_column, .scope_generation = 0, + .vec_len = 16 * 8, // TODO: set this per cpu }; defer { function.frame_allocs.deinit(gpa); @@ -1040,10 +1045,60 @@ pub fn addExtraAssumeCapacity(func: *Func, extra: anytype) u32 { return result; } +/// Returns a temporary register that contains the value of the `reg` csr. +/// +/// Caller's duty to lock the return register is needed. +fn getCsr(func: *Func, csr: CSR) !Register { + assert(func.hasFeature(.zicsr)); + const dst_reg = try func.register_manager.allocReg(null, func.regTempClassForType(Type.usize)); + _ = try func.addInst(.{ + .tag = .csrrs, + .ops = .csr, + .data = .{ + .csr = .{ + .csr = csr, + .rd = dst_reg, + .rs1 = .x0, + }, + }, + }); + return dst_reg; +} + +fn setVl(func: *Func, dst_reg: Register, avl: u5, options: bits.VType) !void { + if (avl == 0) { + const options_int: u12 = @as(u12, 0) | @as(u8, @bitCast(options)); + _ = try func.addInst(.{ + .tag = .vsetvli, + .ops = .rri, + .data = .{ .i_type = .{ + .rd = dst_reg, + .rs1 = .zero, + .imm12 = Immediate.u(options_int), + } }, + }); + } else { + const options_int: u12 = (~@as(u12, 0) << 10) | @as(u8, @bitCast(options)); + _ = try func.addInst(.{ + .tag = .vsetivli, + .ops = .rri, + .data = .{ + .i_type = .{ + .rd = dst_reg, + .rs1 = @enumFromInt(avl), + .imm12 = Immediate.u(options_int), + }, + }, + }); + } +} + const required_features = [_]Target.riscv.Feature{ .d, .m, .a, + .zicsr, + .v, }; fn gen(func: *Func) !void { @@ -1101,7 +1156,19 @@ fn gen(func: *Func) !void { const backpatch_ra_restore = try func.addPseudo(.pseudo_dead); const backpatch_fp_restore = try func.addPseudo(.pseudo_dead); const backpatch_stack_alloc_restore = try func.addPseudo(.pseudo_dead); - try func.addPseudoNone(.pseudo_ret); + + // ret + _ = try func.addInst(.{ + .tag = .jalr, + .ops = .rri, + .data = .{ + .i_type = .{ + .rd = .zero, + .rs1 = .ra, + .imm12 = Immediate.s(0), + }, + }, + }); const frame_layout = try func.computeFrameLayout(); const need_save_reg = frame_layout.save_reg_list.count() > 0; @@ -1842,8 +1909,8 @@ fn typeRegClass(func: *Func, ty: Type) abi.RegisterClass { const zcu = pt.zcu; return switch (ty.zigTypeTag(zcu)) { .Float => .float, - .Vector => @panic("TODO: typeRegClass for Vectors"), - inline else => .int, + .Vector => .vector, + else => .int, }; } @@ -1852,7 +1919,7 @@ fn regGeneralClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet const zcu = pt.zcu; return switch (ty.zigTypeTag(zcu)) { .Float => abi.Registers.Float.general_purpose, - .Vector => @panic("TODO: regGeneralClassForType for Vectors"), + .Vector => abi.Registers.Vector.general_purpose, else => abi.Registers.Integer.general_purpose, }; } @@ -1862,7 +1929,7 @@ fn regTempClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet { const zcu = pt.zcu; return switch (ty.zigTypeTag(zcu)) { .Float => abi.Registers.Float.temporary, - .Vector => @panic("TODO: regTempClassForType for Vectors"), + .Vector => abi.Registers.Vector.general_purpose, // there are no temporary vector registers else => abi.Registers.Integer.temporary, }; } @@ -1870,20 +1937,19 @@ fn regTempClassForType(func: *Func, ty: Type) RegisterManager.RegisterBitSet { fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue { const pt = func.pt; - const abi_size = math.cast(u32, elem_ty.abiSize(pt)) orelse { - return func.fail("type '{}' too big to fit into stack frame", .{elem_ty.fmt(pt)}); + const bit_size = elem_ty.bitSize(pt); + const min_size: u64 = switch (elem_ty.zigTypeTag(pt.zcu)) { + .Float => if (func.hasFeature(.d)) 64 else 32, + .Vector => func.vec_len, + else => 64, }; - const min_size: u32 = switch (elem_ty.zigTypeTag(pt.zcu)) { - .Float => 4, - .Vector => @panic("allocRegOrMem Vector"), - else => 8, - }; - - if (reg_ok and abi_size <= min_size) { + if (reg_ok and bit_size <= min_size) { if (func.register_manager.tryAllocReg(inst, func.regGeneralClassForType(elem_ty))) |reg| { return .{ .register = reg }; } + } else if (reg_ok and elem_ty.zigTypeTag(pt.zcu) == .Vector) { + return func.fail("did you forget to extend vector registers before allocating", .{}); } const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(elem_ty, pt)); @@ -1896,10 +1962,13 @@ fn allocRegOrMem(func: *Func, elem_ty: Type, inst: ?Air.Inst.Index, reg_ok: bool fn allocReg(func: *Func, reg_class: abi.RegisterClass) !struct { Register, RegisterLock } { if (reg_class == .float and !func.hasFeature(.f)) std.debug.panic("allocReg class == float where F isn't enabled", .{}); + if (reg_class == .vector and !func.hasFeature(.v)) + std.debug.panic("allocReg class == vector where V isn't enabled", .{}); const class = switch (reg_class) { .int => abi.Registers.Integer.general_purpose, .float => abi.Registers.Float.general_purpose, + .vector => abi.Registers.Vector.general_purpose, }; const reg = try func.register_manager.allocReg(null, class); @@ -1915,7 +1984,8 @@ fn promoteReg(func: *Func, ty: Type, operand: MCValue) !struct { Register, ?Regi return .{ op_reg, func.register_manager.lockReg(operand.register) }; } - const reg, const lock = try func.allocReg(func.typeRegClass(ty)); + const class = func.typeRegClass(ty); + const reg, const lock = try func.allocReg(class); try func.genSetReg(ty, reg, operand); return .{ reg, lock }; } @@ -2372,6 +2442,42 @@ fn genBinOp( }, }); }, + .Vector => { + const mir_tag: Mir.Inst.Tag = switch (tag) { + .add => .vaddvv, + else => return func.fail("TODO: genBinOp {s} Vector", .{@tagName(tag)}), + }; + + const num_elem: u5 = math.cast(u5, lhs_ty.vectorLen(zcu)) orelse { + return func.fail("TODO: genBinOp use vsetvli for larger avl sizes", .{}); + }; + const elem_size = lhs_ty.childType(zcu).bitSize(pt); + + try func.setVl(.zero, num_elem, .{ + .vlmul = .mf2, + .vsew = switch (elem_size) { + 8 => .@"8", + 16 => .@"16", + 32 => .@"32", + 64 => .@"64", + else => unreachable, + }, + .vma = true, + .vta = true, + }); + + _ = try func.addInst(.{ + .tag = mir_tag, + .ops = .rrr, + .data = .{ + .r_type = .{ + .rd = dst_reg, + .rs1 = lhs_reg, + .rs2 = rhs_reg, + }, + }, + }); + }, else => unreachable, } }, @@ -3560,13 +3666,12 @@ fn airSetUnionTag(func: *Func, inst: Air.Inst.Index) !void { } fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.bin_file.comp.module.?; - const mod = func.bin_file.comp.module.?; + const pt = func.pt; const ty_op = func.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const tag_ty = func.typeOfIndex(inst); const union_ty = func.typeOf(ty_op.operand); - const layout = union_ty.unionGetLayout(mod); + const layout = union_ty.unionGetLayout(pt); if (layout.tag_size == 0) { return func.finishAir(inst, .none, .{ ty_op.operand, .none, .none }); @@ -3577,7 +3682,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { const frame_mcv = try func.allocRegOrMem(union_ty, null, false); try func.genCopy(union_ty, frame_mcv, operand); - const tag_abi_size = tag_ty.abiSize(mod); + const tag_abi_size = tag_ty.abiSize(pt); const result_reg, const result_lock = try func.allocReg(.int); defer func.register_manager.unlockReg(result_lock); @@ -3597,7 +3702,7 @@ fn airGetUnionTag(func: *Func, inst: Air.Inst.Index) !void { } else { return func.fail( "TODO implement get_union_tag for ABI larger than 8 bytes and operand {}, tag {}", - .{ frame_mcv, tag_ty.fmt(zcu) }, + .{ frame_mcv, tag_ty.fmt(pt) }, ); } }, @@ -3781,13 +3886,13 @@ fn airBitReverse(func: *Func, inst: Air.Inst.Index) !void { } fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { - const zcu = func.bin_file.comp.module.?; + const pt = func.pt; const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else result: { const ty = func.typeOf(un_op); const operand = try func.resolveInst(un_op); - const operand_bit_size = ty.bitSize(zcu); + const operand_bit_size = ty.bitSize(pt); if (!math.isPowerOfTwo(operand_bit_size)) return func.fail("TODO: airUnaryMath non-pow 2", .{}); @@ -3799,7 +3904,7 @@ fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { const dst_reg, const dst_lock = try func.allocReg(dst_class); defer func.register_manager.unlockReg(dst_lock); - switch (ty.zigTypeTag(zcu)) { + switch (ty.zigTypeTag(pt.zcu)) { .Float => { assert(dst_class == .float); @@ -3833,7 +3938,7 @@ fn airUnaryMath(func: *Func, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { else => return func.fail("TODO: airUnaryMath Float {s}", .{@tagName(tag)}), } }, - else => return func.fail("TODO: airUnaryMath ty: {}", .{ty.fmt(zcu)}), + else => return func.fail("TODO: airUnaryMath ty: {}", .{ty.fmt(pt)}), } break :result MCValue{ .register = dst_reg }; @@ -4510,7 +4615,27 @@ fn airRet(func: *Func, inst: Air.Inst.Index, safety: bool) !void { .none => {}, .register, .register_pair, - => try func.genCopy(ret_ty, func.ret_mcv.short, .{ .air_ref = un_op }), + => { + if (ret_ty.isVector(zcu)) { + const bit_size = ret_ty.totalVectorBits(pt); + + // set the vtype to hold the entire vector's contents in a single element + try func.setVl(.zero, 0, .{ + .vsew = switch (bit_size) { + 8 => .@"8", + 16 => .@"16", + 32 => .@"32", + 64 => .@"64", + else => unreachable, + }, + .vlmul = .m1, + .vma = true, + .vta = true, + }); + } + + try func.genCopy(ret_ty, func.ret_mcv.short, .{ .air_ref = un_op }); + }, .indirect => |reg_off| { try func.register_manager.getReg(reg_off.reg, null); const lock = func.register_manager.lockRegAssumeUnused(reg_off.reg); @@ -5735,7 +5860,12 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! const zcu = pt.zcu; const abi_size: u32 = @intCast(ty.abiSize(pt)); - if (abi_size > 8) return std.debug.panic("tried to set reg with size {}", .{abi_size}); + const max_size: u32 = switch (reg.class()) { + .int => 64, + .float => if (func.hasFeature(.d)) 64 else 32, + .vector => func.vec_len, + }; + if (abi_size > max_size) return std.debug.panic("tried to set reg with size {}", .{abi_size}); const dst_reg_class = reg.class(); switch (src_mcv) { @@ -5835,13 +5965,6 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! if (src_reg.id() == reg.id()) return; - const src_reg_class = src_reg.class(); - - if (src_reg_class == .float and dst_reg_class == .int) { - // to move from float -> int, we use FMV.X.W - return func.fail("TODO: genSetReg float -> int", .{}); - } - // mv reg, src_reg _ = try func.addInst(.{ .tag = .pseudo, @@ -5854,6 +5977,43 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! }, .register_pair => return func.fail("genSetReg should we allow reg -> reg_pair?", .{}), .load_frame => |frame| { + if (reg.class() == .vector) { + if (abi_size > 8) + return func.fail("TODO: genSetReg vectors > 8", .{}); + + const temp_reg = try func.register_manager.allocReg(null, abi.Registers.Integer.temporary); + const temp_lock = func.register_manager.lockRegAssumeUnused(temp_reg); + defer func.register_manager.unlockReg(temp_lock); + + try func.setVl(.zero, 1, .{ + .vsew = switch (abi_size) { + 1 => .@"8", + 2 => .@"16", + 4 => .@"32", + 8 => .@"64", + else => unreachable, + }, + .vlmul = .m1, + .vma = true, + .vta = true, + }); + + try func.genCopy(ty, .{ .register = temp_reg }, .{ .load_frame = frame }); + + _ = try func.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_mv, + .data = .{ + .rr = .{ + .rd = reg, + .rs = temp_reg, + }, + }, + }); + + return; + } + _ = try func.addInst(.{ .tag = .pseudo, .ops = .pseudo_load_rm, @@ -6195,7 +6355,7 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void { } fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { - const zcu = func.pt.zcu; + const pt = func.pt; const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; @@ -6206,13 +6366,13 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { const ptr_mcv = try func.resolveInst(pl_op.operand); const val_ty = func.typeOf(extra.operand); - const val_size = val_ty.abiSize(func.pt); + const val_size = val_ty.abiSize(pt); const val_mcv = try func.resolveInst(extra.operand); if (!math.isPowerOfTwo(val_size)) return func.fail("TODO: airAtomicRmw non-pow 2", .{}); - switch (val_ty.zigTypeTag(zcu)) { + switch (val_ty.zigTypeTag(pt.zcu)) { .Int => {}, inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}), else => unreachable, @@ -6735,15 +6895,15 @@ fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking { } fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { - const zcu = func.pt.zcu; + const pt = func.pt; const gpa = func.gpa; - const owner_decl_index = zcu.funcOwnerDeclIndex(func.func_index); + const owner_decl_index = pt.zcu.funcOwnerDeclIndex(func.func_index); const lf = func.bin_file; const src_loc = func.src_loc; - if (val.isUndef(zcu)) { - const local_sym_index = lf.lowerUnnamedConst(func.pt, val, owner_decl_index) catch |err| { + if (val.isUndef(pt.zcu)) { + const local_sym_index = lf.lowerUnnamedConst(pt, val, owner_decl_index) catch |err| { const msg = try ErrorMsg.create(gpa, src_loc, "lowering unnamed undefined constant failed: {s}", .{@errorName(err)}); func.err_msg = msg; return error.CodegenFail; @@ -6760,7 +6920,7 @@ fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { const result = try codegen.genTypedValue( lf, - func.pt, + pt, src_loc, val, owner_decl_index, diff --git a/src/arch/riscv64/Encoding.zig b/src/arch/riscv64/Encoding.zig index 5f83e3de21..2d285419d1 100644 --- a/src/arch/riscv64/Encoding.zig +++ b/src/arch/riscv64/Encoding.zig @@ -11,6 +11,7 @@ const OpCode = enum(u7) { STORE = 0b0100011, STORE_FP = 0b0100111, AMO = 0b0101111, + OP_V = 0b1010111, OP = 0b0110011, OP_32 = 0b0111011, LUI = 0b0110111, @@ -83,9 +84,51 @@ const Enc = struct { funct3: u3, has_5: bool, }, + vecls: struct { + width: VecWidth, + umop: Umop, + vm: bool, + mop: Mop, + mew: bool, + nf: u3, + }, + vecmath: struct { + vm: bool, + funct6: u6, + funct3: VecType, + }, /// U-type none, }, + + const Mop = enum(u2) { + unit = 0b00, + unord = 0b01, + stride = 0b10, + ord = 0b11, + }; + + const Umop = enum(u5) { + unit = 0b00000, + whole = 0b01000, + mask = 0b01011, + fault = 0b10000, + }; + + const VecWidth = enum(u3) { + @"32" = 0b110, + @"64" = 0b111, + }; + + const VecType = enum(u3) { + OPIVV = 0b000, + OPFVV = 0b001, + OPMVV = 0b010, + OPIVI = 0b011, + OPIVX = 0b100, + OPFVF = 0b101, + OPMVX = 0b110, + }; }; pub const Mnemonic = enum { @@ -115,6 +158,9 @@ pub const Mnemonic = enum { addi, jalr, + vsetivli, + vsetvli, + // U Type lui, auipc, @@ -155,6 +201,8 @@ pub const Mnemonic = enum { ebreak, unimp, + csrrs, + // M extension mul, mulw, @@ -217,6 +265,17 @@ pub const Mnemonic = enum { fsgnjnd, fsgnjxd, + // V Extension + vle32v, + vle64v, + + vse32v, + vse64v, + + vaddvv, + vadcxv, + vadcvx, + // MISC fence, fencetso, @@ -373,6 +432,9 @@ pub const Mnemonic = enum { .flw => .{ .opcode = .LOAD_FP, .data = .{ .f = .{ .funct3 = 0b010 } } }, .fld => .{ .opcode = .LOAD_FP, .data = .{ .f = .{ .funct3 = 0b011 } } }, + + .vle32v => .{ .opcode = .LOAD_FP, .data = .{ .vecls = .{ .width = .@"32", .umop = .unit, .vm = true, .mop = .unit, .mew = true, .nf = 0b000 } } }, + .vle64v => .{ .opcode = .LOAD_FP, .data = .{ .vecls = .{ .width = .@"64", .umop = .unit, .vm = true, .mop = .unit, .mew = true, .nf = 0b000 } } }, // STORE_FP @@ -380,6 +442,8 @@ pub const Mnemonic = enum { .fsw => .{ .opcode = .STORE_FP, .data = .{ .f = .{ .funct3 = 0b010 } } }, .fsd => .{ .opcode = .STORE_FP, .data = .{ .f = .{ .funct3 = 0b011 } } }, + .vse32v => .{ .opcode = .STORE_FP, .data = .{ .vecls = .{ .width = .@"32", .umop = .unit, .vm = true, .mop = .unit, .mew = true, .nf = 0b000 } } }, + .vse64v => .{ .opcode = .STORE_FP, .data = .{ .vecls = .{ .width = .@"64", .umop = .unit, .vm = true, .mop = .unit, .mew = true, .nf = 0b000 } } }, // JALR @@ -410,6 +474,8 @@ pub const Mnemonic = enum { .ecall => .{ .opcode = .SYSTEM, .data = .{ .f = .{ .funct3 = 0b000 } } }, .ebreak => .{ .opcode = .SYSTEM, .data = .{ .f = .{ .funct3 = 0b000 } } }, + + .csrrs => .{ .opcode = .SYSTEM, .data = .{ .f = .{ .funct3 = 0b010 } } }, // NONE @@ -449,7 +515,13 @@ pub const Mnemonic = enum { .amominud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11000 } } }, .amomaxud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } }, - + // OP_V + .vsetivli => .{ .opcode = .OP_V, .data = .{ .f = .{ .funct3 = 0b111 } } }, + .vsetvli => .{ .opcode = .OP_V, .data = .{ .f = .{ .funct3 = 0b111 } } }, + .vaddvv => .{ .opcode = .OP_V, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b000000, .funct3 = .OPIVV } } }, + .vadcxv => .{ .opcode = .OP_V, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b010000, .funct3 = .OPMVX } } }, + .vadcvx => .{ .opcode = .OP_V, .data = .{ .vecmath = .{ .vm = true, .funct6 = 0b010000, .funct3 = .OPMVV } } }, + // zig fmt: on }; } @@ -465,7 +537,6 @@ pub const InstEnc = enum { J, fence, amo, - /// extras that have unusual op counts system, pub fn fromMnemonic(mnem: Mnemonic) InstEnc { @@ -494,6 +565,10 @@ pub const InstEnc = enum { .flw, .fld, + + .csrrs, + .vsetivli, + .vsetvli, => .I, .lui, @@ -587,6 +662,14 @@ pub const InstEnc = enum { .fsgnjxs, .fsgnjxd, + + .vle32v, + .vle64v, + .vse32v, + .vse64v, + .vaddvv, + .vadcxv, + .vadcvx, => .R, .ecall, @@ -757,6 +840,25 @@ pub const Data = union(InstEnc) { }, }; }, + .csrrs => { + assert(ops.len == 3); + + const csr = ops[0].csr; + const rs1 = ops[1].reg; + const rd = ops[2].reg; + + return .{ + .I = .{ + .rd = rd.encodeId(), + .rs1 = rs1.encodeId(), + + .imm0_11 = @intFromEnum(csr), + + .opcode = @intFromEnum(enc.opcode), + .funct3 = enc.data.f.funct3, + }, + }; + }, else => {}, } @@ -783,6 +885,25 @@ pub const Data = union(InstEnc) { .funct3 = fmt.rm, .funct7 = (@as(u7, fmt.funct5) << 2) | @intFromEnum(fmt.fmt), }, + .vecls => |vec| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + + .rs2 = @intFromEnum(vec.umop), + + .opcode = @intFromEnum(enc.opcode), + .funct3 = @intFromEnum(vec.width), + .funct7 = (@as(u7, vec.nf) << 4) | (@as(u7, @intFromBool(vec.mew)) << 3) | (@as(u7, @intFromEnum(vec.mop)) << 1) | @intFromBool(vec.vm), + }, + .vecmath => |vec| .{ + .rd = ops[0].reg.encodeId(), + .rs1 = ops[1].reg.encodeId(), + .rs2 = ops[2].reg.encodeId(), + + .opcode = @intFromEnum(enc.opcode), + .funct3 = @intFromEnum(vec.funct3), + .funct7 = (@as(u7, vec.funct6) << 1) | @intFromBool(vec.vm), + }, else => unreachable, }, }; @@ -897,21 +1018,21 @@ pub const Data = union(InstEnc) { .amo => { assert(ops.len == 5); - const rd = ops[0]; - const rs1 = ops[1]; - const rs2 = ops[2]; - const rl = ops[3]; - const aq = ops[4]; + const rd = ops[0].reg; + const rs1 = ops[1].reg; + const rs2 = ops[2].reg; + const rl = ops[3].barrier; + const aq = ops[4].barrier; return .{ .amo = .{ - .rd = rd.reg.encodeId(), - .rs1 = rs1.reg.encodeId(), - .rs2 = rs2.reg.encodeId(), + .rd = rd.encodeId(), + .rs1 = rs1.encodeId(), + .rs2 = rs2.encodeId(), // TODO: https://github.com/ziglang/zig/issues/20113 - .rl = if (rl.barrier == .rl) true else false, - .aq = if (aq.barrier == .aq) true else false, + .rl = if (rl == .rl) true else false, + .aq = if (aq == .aq) true else false, .opcode = @intFromEnum(enc.opcode), .funct3 = @intFromEnum(enc.data.amo.width), @@ -919,7 +1040,6 @@ pub const Data = union(InstEnc) { }, }; }, - else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}), } } diff --git a/src/arch/riscv64/Lower.zig b/src/arch/riscv64/Lower.zig index 723c64b160..9663932b76 100644 --- a/src/arch/riscv64/Lower.zig +++ b/src/arch/riscv64/Lower.zig @@ -80,58 +80,99 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct { .pseudo_load_rm => { const dest_reg = rm.r; const dest_reg_class = dest_reg.class(); - const float = dest_reg_class == .float; const src_size = rm.m.mod.size; const unsigned = rm.m.mod.unsigned; - const tag: Encoding.Mnemonic = if (!float) - switch (src_size) { + const tag: Encoding.Mnemonic = switch (dest_reg_class) { + .int => switch (src_size) { .byte => if (unsigned) .lbu else .lb, .hword => if (unsigned) .lhu else .lh, .word => if (unsigned) .lwu else .lw, .dword => .ld, - } - else switch (src_size) { - .byte => unreachable, // Zig does not support 8-bit floats - .hword => return lower.fail("TODO: lowerMir pseudo_load_rm support 16-bit floats", .{}), - .word => .flw, - .dword => .fld, + }, + .float => switch (src_size) { + .byte => unreachable, // Zig does not support 8-bit floats + .hword => return lower.fail("TODO: lowerMir pseudo_load_rm support 16-bit floats", .{}), + .word => .flw, + .dword => .fld, + }, + .vector => switch (src_size) { + .byte, + .hword, + => return lower.fail( + "TODO: lowerMir pseudo_load_rm support {s} vector", + .{@tagName(src_size)}, + ), + .word => .vle32v, + .dword => .vle64v, + }, }; - try lower.emit(tag, &.{ - .{ .reg = rm.r }, - .{ .reg = frame_loc.base }, - .{ .imm = Immediate.s(frame_loc.disp) }, - }); + switch (dest_reg_class) { + .int, .float => { + try lower.emit(tag, &.{ + .{ .reg = rm.r }, + .{ .reg = frame_loc.base }, + .{ .imm = Immediate.s(frame_loc.disp) }, + }); + }, + .vector => { + try lower.emit(tag, &.{ + .{ .reg = rm.r }, + .{ .reg = frame_loc.base }, + .{ .reg = .x0 }, + }); + }, + } }, .pseudo_store_rm => { const src_reg = rm.r; const src_reg_class = src_reg.class(); - const float = src_reg_class == .float; - // TODO: do we actually need this? are all stores not usize? const dest_size = rm.m.mod.size; - const tag: Encoding.Mnemonic = if (!float) - switch (dest_size) { + const tag: Encoding.Mnemonic = switch (src_reg_class) { + .int => switch (dest_size) { .byte => .sb, .hword => .sh, .word => .sw, .dword => .sd, - } - else switch (dest_size) { - .byte => unreachable, // Zig does not support 8-bit floats - .hword => return lower.fail("TODO: lowerMir pseudo_load_rm support 16-bit floats", .{}), - .word => .fsw, - .dword => .fsd, + }, + .float => switch (dest_size) { + .byte => unreachable, // Zig does not support 8-bit floats + .hword => return lower.fail("TODO: lowerMir pseudo_store_rm support 16-bit floats", .{}), + .word => .fsw, + .dword => .fsd, + }, + .vector => switch (dest_size) { + .byte, + .hword, + => return lower.fail( + "TODO: lowerMir pseudo_load_rm support {s} vector", + .{@tagName(dest_size)}, + ), + .word => .vse32v, + .dword => .vse64v, + }, }; - try lower.emit(tag, &.{ - .{ .reg = frame_loc.base }, - .{ .reg = rm.r }, - .{ .imm = Immediate.s(frame_loc.disp) }, - }); + switch (src_reg_class) { + .int, .float => { + try lower.emit(tag, &.{ + .{ .reg = frame_loc.base }, + .{ .reg = rm.r }, + .{ .imm = Immediate.s(frame_loc.disp) }, + }); + }, + .vector => { + try lower.emit(tag, &.{ + .{ .reg = frame_loc.base }, + .{ .reg = rm.r }, + .{ .reg = .x0 }, + }); + }, + } }, else => unreachable, } @@ -143,34 +184,47 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct { const dst_class = rr.rd.class(); const src_class = rr.rs.class(); - assert(dst_class == src_class); - - switch (dst_class) { - .float => { - try lower.emit(if (lower.hasFeature(.d)) .fsgnjnd else .fsgnjns, &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - .{ .reg = rr.rs }, - }); + switch (src_class) { + .float => switch (dst_class) { + .float => { + try lower.emit(if (lower.hasFeature(.d)) .fsgnjnd else .fsgnjns, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .reg = rr.rs }, + }); + }, + .int, .vector => return lower.fail("TODO: lowerMir pseudo_mv float -> {s}", .{@tagName(dst_class)}), }, - .int => { - try lower.emit(.addi, &.{ - .{ .reg = rr.rd }, - .{ .reg = rr.rs }, - .{ .imm = Immediate.s(0) }, - }); + .int => switch (dst_class) { + .int => { + try lower.emit(.addi, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .imm = Immediate.s(0) }, + }); + }, + .vector => { + try lower.emit(.vadcxv, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .reg = .zero }, + }); + }, + .float => return lower.fail("TODO: lowerMir pseudo_mv int -> {s}", .{@tagName(dst_class)}), + }, + .vector => switch (dst_class) { + .int => { + try lower.emit(.vadcvx, &.{ + .{ .reg = rr.rd }, + .{ .reg = rr.rs }, + .{ .reg = .zero }, + }); + }, + .float, .vector => return lower.fail("TODO: lowerMir pseudo_mv vector -> {s}", .{@tagName(dst_class)}), }, } }, - .pseudo_ret => { - try lower.emit(.jalr, &.{ - .{ .reg = .zero }, - .{ .reg = .ra }, - .{ .imm = Immediate.s(0) }, - }); - }, - .pseudo_j => { try lower.emit(.jal, &.{ .{ .reg = .zero }, @@ -209,7 +263,10 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct { const rm = inst.data.rm; assert(rm.r.class() == .int); - const frame = rm.m.toFrameLoc(lower.mir); + const frame: Mir.FrameLoc = if (options.allow_frame_locs) + rm.m.toFrameLoc(lower.mir) + else + .{ .base = .s0, .disp = 0 }; try lower.emit(.addi, &.{ .{ .reg = rm.r }, @@ -376,6 +433,7 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct { }); }, }, + .vector => return lower.fail("TODO: lowerMir pseudo_cmp vector", .{}), } }, @@ -497,6 +555,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { .{ .reg = inst.data.r_type.rs1 }, .{ .reg = inst.data.r_type.rs2 }, }, + .csr => &.{ + .{ .csr = inst.data.csr.csr }, + .{ .reg = inst.data.csr.rs1 }, + .{ .reg = inst.data.csr.rd }, + }, else => return lower.fail("TODO: generic lower ops {s}", .{@tagName(inst.ops)}), }); } @@ -523,17 +586,22 @@ fn pushPopRegList(lower: *Lower, comptime spilling: bool, reg_list: Mir.Register while (it.next()) |i| { const frame = lower.mir.frame_locs.get(@intFromEnum(bits.FrameIndex.spill_frame)); const reg = abi.Registers.all_preserved[i]; + const reg_class = reg.class(); - const is_float_reg = reg_class == .float; + const load_inst: Encoding.Mnemonic, const store_inst: Encoding.Mnemonic = switch (reg_class) { + .int => .{ .ld, .sd }, + .float => .{ .fld, .fsd }, + .vector => unreachable, + }; if (spilling) { - try lower.emit(if (is_float_reg) .fsd else .sd, &.{ + try lower.emit(store_inst, &.{ .{ .reg = frame.base }, .{ .reg = abi.Registers.all_preserved[i] }, .{ .imm = Immediate.s(frame.disp + reg_i) }, }); } else { - try lower.emit(if (is_float_reg) .fld else .ld, &.{ + try lower.emit(load_inst, &.{ .{ .reg = abi.Registers.all_preserved[i] }, .{ .reg = frame.base }, .{ .imm = Immediate.s(frame.disp + reg_i) }, diff --git a/src/arch/riscv64/Mir.zig b/src/arch/riscv64/Mir.zig index da872da067..24b83e6381 100644 --- a/src/arch/riscv64/Mir.zig +++ b/src/arch/riscv64/Mir.zig @@ -134,7 +134,16 @@ pub const Inst = struct { fltd, fled, - /// A Extension Instructions + // Zicsr Extension Instructions + csrrs, + + // V Extension Instructions + vsetvli, + vsetivli, + vsetvl, + vaddvv, + + // A Extension Instructions amo, /// A pseudo-instruction. Used for anything that isn't 1:1 with an @@ -146,91 +155,57 @@ pub const Inst = struct { /// this union. `Ops` determines which union field is active, as well as /// how to interpret the data within. pub const Data = union { - /// No additional data - /// - /// Used by e.g. ebreak nop: void, - /// Another instruction. - /// - /// Used by e.g. b inst: Index, - /// Index into `extra`. Meaning of what can be found there is context-dependent. - /// - /// Used by e.g. load_memory payload: u32, - r_type: struct { rd: Register, rs1: Register, rs2: Register, }, - i_type: struct { rd: Register, rs1: Register, imm12: Immediate, }, - s_type: struct { rs1: Register, rs2: Register, imm5: Immediate, imm7: Immediate, }, - b_type: struct { rs1: Register, rs2: Register, inst: Inst.Index, }, - u_type: struct { rd: Register, imm20: Immediate, }, - j_type: struct { rd: Register, inst: Inst.Index, }, - - /// Debug info: line and column - /// - /// Used by e.g. pseudo_dbg_line pseudo_dbg_line_column: struct { line: u32, column: u32, }, - - // Custom types to be lowered - - /// Register + Memory rm: struct { r: Register, m: Memory, }, - reg_list: Mir.RegisterList, - - /// A register - /// - /// Used by e.g. blr reg: Register, - - /// Two registers - /// - /// Used by e.g. mv rr: struct { rd: Register, rs: Register, }, - fabs: struct { rd: Register, rs: Register, bits: u16, }, - compare: struct { rd: Register, rs1: Register, @@ -245,12 +220,10 @@ pub const Inst = struct { }, ty: Type, }, - reloc: struct { atom_index: u32, sym_index: u32, }, - fence: struct { pred: Barrier, succ: Barrier, @@ -259,7 +232,6 @@ pub const Inst = struct { tso, }, }, - amo: struct { rd: Register, rs1: Register, @@ -269,6 +241,11 @@ pub const Inst = struct { op: AmoOp, ty: Type, }, + csr: struct { + csr: CSR, + rs1: Register, + rd: Register, + }, }; pub const Ops = enum { @@ -293,6 +270,9 @@ pub const Inst = struct { /// Another instruction. inst, + /// Control and Status Register Instruction. + csr, + /// Pseudo-instruction that will generate a backpatched /// function prologue. pseudo_prologue, @@ -321,11 +301,6 @@ pub const Inst = struct { /// Uses `rm` payload. pseudo_lea_rm, - /// Shorthand for returning, aka jumping to ra register. - /// - /// Uses nop payload. - pseudo_ret, - /// Jumps. Uses `inst` payload. pseudo_j, @@ -363,14 +338,6 @@ pub const Inst = struct { pseudo_amo, }; - // Make sure we don't accidentally make instructions bigger than expected. - // Note that in Debug builds, Zig is allowed to insert a secret field for safety checks. - // comptime { - // if (builtin.mode != .Debug) { - // assert(@sizeOf(Inst) == 8); - // } - // } - pub fn format( inst: Inst, comptime fmt: []const u8, @@ -490,6 +457,7 @@ const assert = std.debug.assert; const bits = @import("bits.zig"); const Register = bits.Register; +const CSR = bits.CSR; const Immediate = bits.Immediate; const Memory = bits.Memory; const FrameIndex = bits.FrameIndex; diff --git a/src/arch/riscv64/abi.zig b/src/arch/riscv64/abi.zig index 0a41b61d79..3f5f7f6744 100644 --- a/src/arch/riscv64/abi.zig +++ b/src/arch/riscv64/abi.zig @@ -193,6 +193,15 @@ pub fn classifySystem(ty: Type, pt: Zcu.PerThread) [8]SystemClass { } return memory_class; }, + .Vector => { + // we pass vectors through integer registers if they are small enough to fit. + const vec_bits = ty.totalVectorBits(pt); + if (vec_bits <= 64) { + result[0] = .integer; + return result; + } + return memory_class; + }, else => |bad_ty| std.debug.panic("classifySystem {s}", .{@tagName(bad_ty)}), } } @@ -254,15 +263,15 @@ fn classifyStruct( } } -const allocatable_registers = Registers.Integer.all_regs ++ Registers.Float.all_regs; +const allocatable_registers = Registers.Integer.all_regs ++ Registers.Float.all_regs ++ Registers.Vector.all_regs; pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); -// Register classes const RegisterBitSet = RegisterManager.RegisterBitSet; pub const RegisterClass = enum { int, float, + vector, }; pub const Registers = struct { @@ -322,6 +331,19 @@ pub const Registers = struct { pub const all_regs = callee_preserved_regs ++ function_arg_regs ++ temporary_regs; }; + + pub const Vector = struct { + pub const general_purpose = initRegBitSet(Integer.all_regs.len + Float.all_regs.len, all_regs.len); + + // zig fmt: off + pub const all_regs = [_]Register{ + .v0, .v1, .v2, .v3, .v4, .v5, .v6, .v7, + .v8, .v9, .v10, .v11, .v12, .v13, .v14, .v15, + .v16, .v17, .v18, .v19, .v20, .v21, .v22, .v23, + .v24, .v25, .v26, .v27, .v28, .v29, .v30, .v31, + }; + // zig fmt: on + }; }; fn initRegBitSet(start: usize, length: usize) RegisterBitSet { diff --git a/src/arch/riscv64/bits.zig b/src/arch/riscv64/bits.zig index 75c93d7f13..07dcf09e8f 100644 --- a/src/arch/riscv64/bits.zig +++ b/src/arch/riscv64/bits.zig @@ -128,6 +128,12 @@ pub const Immediate = union(enum) { } }; +pub const CSR = enum(u12) { + vl = 0xC20, + vtype = 0xC21, + vlenb = 0xC22, +}; + pub const Register = enum(u8) { // zig fmt: off @@ -169,6 +175,13 @@ pub const Register = enum(u8) { f16, f17, f18, f19, f20, f21, f22, f23, f24, f25, f26, f27, f28, f29, f30, f31, + + // V extension registers + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, + // zig fmt: on /// in RISC-V registers are stored as 5 bit IDs and a register can have @@ -180,11 +193,12 @@ pub const Register = enum(u8) { /// The goal of this function is to return the same ID for `zero` and `x0` but two /// seperate IDs for `x0` and `f0`. We will assume that each register set has 32 registers /// and is repeated twice, once for the named version, once for the number version. - pub fn id(reg: Register) u7 { + pub fn id(reg: Register) u8 { const base = switch (@intFromEnum(reg)) { // zig fmt: off @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => @intFromEnum(Register.zero), @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => @intFromEnum(Register.ft0), + @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => @intFromEnum(Register.v0), else => unreachable, // zig fmt: on }; @@ -207,6 +221,7 @@ pub const Register = enum(u8) { // zig fmt: off @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => 64, @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => if (Target.riscv.featureSetHas(features, .d)) 64 else 32, + @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => 1024, // TODO: look at suggestVectorSize else => unreachable, // zig fmt: on }; @@ -217,6 +232,7 @@ pub const Register = enum(u8) { // zig fmt: off @intFromEnum(Register.zero) ... @intFromEnum(Register.x31) => .int, @intFromEnum(Register.ft0) ... @intFromEnum(Register.f31) => .float, + @intFromEnum(Register.v0) ... @intFromEnum(Register.v31) => .vector, else => unreachable, // zig fmt: on }; @@ -272,3 +288,27 @@ pub const Symbol = struct { /// Index into the linker's symbol table. sym_index: u32, }; + +pub const VType = packed struct(u8) { + vlmul: VlMul, + vsew: VSew, + vta: bool, + vma: bool, +}; + +const VSew = enum(u3) { + @"8" = 0b000, + @"16" = 0b001, + @"32" = 0b010, + @"64" = 0b011, +}; + +const VlMul = enum(u3) { + mf8 = 0b101, + mf4 = 0b110, + mf2 = 0b111, + m1 = 0b000, + m2 = 0b001, + m4 = 0b010, + m8 = 0b011, +}; diff --git a/src/arch/riscv64/encoder.zig b/src/arch/riscv64/encoder.zig index 5962121438..2ef5ba03ec 100644 --- a/src/arch/riscv64/encoder.zig +++ b/src/arch/riscv64/encoder.zig @@ -5,6 +5,7 @@ pub const Instruction = struct { pub const Operand = union(enum) { none, reg: Register, + csr: CSR, mem: Memory, imm: Immediate, barrier: Mir.Barrier, @@ -58,6 +59,7 @@ pub const Instruction = struct { .imm => |imm| try writer.print("{d}", .{imm.asSigned(64)}), .mem => try writer.writeAll("mem"), .barrier => |barrier| try writer.writeAll(@tagName(barrier)), + .csr => |csr| try writer.writeAll(@tagName(csr)), } } } @@ -71,6 +73,7 @@ const bits = @import("bits.zig"); const Encoding = @import("Encoding.zig"); const Register = bits.Register; +const CSR = bits.CSR; const Memory = bits.Memory; const Immediate = bits.Immediate; diff --git a/test/tests.zig b/test/tests.zig index 0862f8deb0..7116233c7e 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -436,11 +436,12 @@ const test_targets = blk: { //}, .{ - .target = .{ - .cpu_arch = .riscv64, - .os_tag = .linux, - .abi = .musl, - }, + .target = std.Target.Query.parse( + .{ + .arch_os_abi = "riscv64-linux-musl", + .cpu_features = "baseline+v", + }, + ) catch @panic("OOM"), .use_llvm = false, .use_lld = false, },