commit a76ce771082c3bf126e9ed745607bc8dadb56c61 (tree)
parent ad1b746e2887d25a6e279762ada516e683db13da
Author: Matthew Lugg <mlugg@mlugg.co.uk>
Date: Fri, 8 May 2026 10:26:10 +0100
llvm: fix lowering of x86 fastcall and vectorcall
LLVM requires these calling conventions to specify `inreg` attributes on
all parameters which are passed via register.
Diffstat:
4 files changed, 165 insertions(+), 73 deletions(-)
diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig
@@ -2718,73 +2718,60 @@ pub const Object = struct {
@panic("TODO: LLVM backend lower async function");
}
- {
- const cc_info = toLlvmCallConv(fn_info.cc, target).?;
+ const cc_info = toLlvmCallConv(fn_info.cc, target).?;
- function_index.setCallConv(cc_info.llvm_cc, &o.builder);
+ function_index.setCallConv(cc_info.llvm_cc, &o.builder);
- if (cc_info.align_stack) {
- try attributes.addFnAttr(.{ .alignstack = .wrap(.fromByteUnits(target.stackAlignment())) }, &o.builder);
- } else {
- _ = try attributes.removeFnAttr(.alignstack);
- }
-
- if (cc_info.naked) {
- try attributes.addFnAttr(.naked, &o.builder);
- } else {
- _ = try attributes.removeFnAttr(.naked);
- }
+ if (cc_info.align_stack) {
+ try attributes.addFnAttr(.{ .alignstack = .wrap(.fromByteUnits(target.stackAlignment())) }, &o.builder);
+ }
- for (0..cc_info.inreg_param_count) |param_idx| {
- try attributes.addParamAttr(param_idx, .inreg, &o.builder);
- }
- for (cc_info.inreg_param_count..std.math.maxInt(u2)) |param_idx| {
- _ = try attributes.removeParamAttr(param_idx, .inreg);
- }
+ if (cc_info.naked) {
+ try attributes.addFnAttr(.naked, &o.builder);
+ }
- switch (fn_info.cc) {
- inline .riscv64_interrupt,
- .riscv32_interrupt,
- .mips_interrupt,
- .mips64_interrupt,
- => |info| {
- try attributes.addFnAttr(.{ .string = .{
- .kind = try o.builder.string("interrupt"),
- .value = try o.builder.string(@tagName(info.mode)),
- } }, &o.builder);
- },
- .arm_interrupt,
- => |info| {
- try attributes.addFnAttr(.{ .string = .{
- .kind = try o.builder.string("interrupt"),
- .value = try o.builder.string(switch (info.type) {
- .generic => "",
- .irq => "IRQ",
- .fiq => "FIQ",
- .swi => "SWI",
- .abort => "ABORT",
- .undef => "UNDEF",
- }),
- } }, &o.builder);
- },
- // these function attributes serve as a backup against any mistakes LLVM makes.
- // clang sets both the function's calling convention and the function attributes
- // in its backend, so future patches to the AVR backend could end up checking only one,
- // possibly breaking our support. it's safer to just emit both.
- .avr_interrupt, .avr_signal, .csky_interrupt => {
- try attributes.addFnAttr(.{ .string = .{
- .kind = try o.builder.string(switch (fn_info.cc) {
- .avr_interrupt,
- .csky_interrupt,
- => "interrupt",
- .avr_signal => "signal",
- else => unreachable,
- }),
- .value = .empty,
- } }, &o.builder);
- },
- else => {},
- }
+ switch (fn_info.cc) {
+ inline .riscv64_interrupt,
+ .riscv32_interrupt,
+ .mips_interrupt,
+ .mips64_interrupt,
+ => |info| {
+ try attributes.addFnAttr(.{ .string = .{
+ .kind = try o.builder.string("interrupt"),
+ .value = try o.builder.string(@tagName(info.mode)),
+ } }, &o.builder);
+ },
+ .arm_interrupt,
+ => |info| {
+ try attributes.addFnAttr(.{ .string = .{
+ .kind = try o.builder.string("interrupt"),
+ .value = try o.builder.string(switch (info.type) {
+ .generic => "",
+ .irq => "IRQ",
+ .fiq => "FIQ",
+ .swi => "SWI",
+ .abort => "ABORT",
+ .undef => "UNDEF",
+ }),
+ } }, &o.builder);
+ },
+ // these function attributes serve as a backup against any mistakes LLVM makes.
+ // clang sets both the function's calling convention and the function attributes
+ // in its backend, so future patches to the AVR backend could end up checking only one,
+ // possibly breaking our support. it's safer to just emit both.
+ .avr_interrupt, .avr_signal, .csky_interrupt => {
+ try attributes.addFnAttr(.{ .string = .{
+ .kind = try o.builder.string(switch (fn_info.cc) {
+ .avr_interrupt,
+ .csky_interrupt,
+ => "interrupt",
+ .avr_signal => "signal",
+ else => unreachable,
+ }),
+ .value = .empty,
+ } }, &o.builder);
+ },
+ else => {},
}
// Function attributes that are independent of analysis results of the function body.
@@ -2821,6 +2808,10 @@ pub const Object = struct {
try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
it.llvm_index += 1;
}
+
+ var remaining_inreg_int = cc_info.inreg_int_params;
+ var remaining_inreg_float = cc_info.inreg_float_params;
+
while (try it.next()) |lowering| switch (lowering) {
.byval => {
const param_index = it.zig_index - 1;
@@ -2828,6 +2819,21 @@ pub const Object = struct {
if (!isByRef(param_ty, zcu)) {
try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
}
+
+ if (remaining_inreg_int > 0 and
+ (param_ty.isPtrAtRuntime(zcu) or
+ (param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
+ {
+ try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
+ remaining_inreg_int -= 1;
+ }
+
+ if (remaining_inreg_float > 0 and
+ param_ty.zigTypeTag(zcu) == .float)
+ {
+ try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
+ remaining_inreg_float -= 1;
+ }
},
.byref => {
const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
@@ -4379,15 +4385,19 @@ const CallingConventionInfo = struct {
align_stack: bool,
/// Whether the function needs a `naked` attribute.
naked: bool,
- /// How many leading parameters to apply the `inreg` attribute to.
- inreg_param_count: u2 = 0,
+ /// How many leading register-sized integer parameters to apply the `inreg` attribute to.
+ inreg_int_params: u2 = 0,
+ /// How many leading floating-point parameters to apply the `inreg` attribute to.
+ inreg_float_params: u3 = 0,
};
pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
- const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
+ const incoming_stack_alignment: ?u64, const inreg_int_params: u2, const inreg_float_params: u3 = switch (cc) {
+ .x86_fastcall => |opts| .{ opts.incoming_stack_alignment, 2, 0 },
+ .x86_vectorcall => |opts| .{ opts.incoming_stack_alignment, 2, 6 },
inline else => |pl| switch (@TypeOf(pl)) {
- void => .{ null, 0 },
+ void => .{ null, 0, 0 },
std.lang.CallingConvention.ArcInterruptOptions,
std.lang.CallingConvention.ArmInterruptOptions,
std.lang.CallingConvention.RiscvInterruptOptions,
@@ -4395,8 +4405,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)
std.lang.CallingConvention.MicroblazeInterruptOptions,
std.lang.CallingConvention.MipsInterruptOptions,
std.lang.CallingConvention.CommonOptions,
- => .{ pl.incoming_stack_alignment, 0 },
- std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
+ => .{ pl.incoming_stack_alignment, 0, 0 },
+ std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params, 0 },
else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
},
};
@@ -4407,7 +4417,8 @@ pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target)
break :need_align a < normal_stack_align;
} else false,
.naked = cc == .naked,
- .inreg_param_count = register_params,
+ .inreg_int_params = inreg_int_params,
+ .inreg_float_params = inreg_float_params,
};
}
pub fn toLlvmCallConvTag(cc_tag: std.lang.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig
@@ -767,11 +767,15 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
},
};
+ const cc_info = llvm.toLlvmCallConv(fn_info.cc, target).?;
+
{
// Add argument attributes.
it = iterateParamTypes(o, fn_info);
it.llvm_index += @intFromBool(sret);
it.llvm_index += @intFromBool(err_return_tracing);
+ var remaining_inreg_int = cc_info.inreg_int_params;
+ var remaining_inreg_float = cc_info.inreg_float_params;
while (try it.next()) |lowering| switch (lowering) {
.byval => {
const param_index = it.zig_index - 1;
@@ -779,6 +783,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
if (!isByRef(param_ty, zcu)) {
try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
}
+
+ if (remaining_inreg_int > 0 and
+ (param_ty.isPtrAtRuntime(zcu) or
+ (param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
+ {
+ try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
+ remaining_inreg_int -= 1;
+ }
+
+ if (remaining_inreg_float > 0 and
+ param_ty.zigTypeTag(zcu) == .float)
+ {
+ try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
+ remaining_inreg_float -= 1;
+ }
},
.byref => {
const param_index = it.zig_index - 1;
@@ -833,7 +852,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
.always_tail => .musttail,
.no_suspend, .always_inline, .compile_time => unreachable,
},
- llvm.toLlvmCallConvTag(fn_info.cc, target).?,
+ cc_info.llvm_cc,
try attributes.finish(&o.builder),
try o.lowerType(zig_fn_ty),
llvm_fn,
diff --git a/test/c_abi/cfuncs.c b/test/c_abi/cfuncs.c
@@ -5892,3 +5892,30 @@ struct byval_tail_callsite_attr_Rect {
double c_byval_tail_callsite_attr(struct byval_tail_callsite_attr_Rect in) {
return in.size.width;
}
+
+#ifdef __i386__
+void __attribute__((fastcall)) zig_fastcall_check(int a, float b, void *c, double d, int e);
+void __attribute__((fastcall)) c_fastcall_check(int a, float b, void *c, double d, int e) {
+ assert_or_panic(a == 1);
+ assert_or_panic(b == 2.0);
+ assert_or_panic((uintptr_t)c == 3);
+ assert_or_panic(d == 4.0);
+ assert_or_panic(e == 5);
+ zig_fastcall_check(a, b, c, d, e);
+}
+
+void __attribute__((vectorcall)) zig_vectorcall_check(int a, float b, double c, void *d, float e, double f, double g, float h, float i, int j);
+void __attribute__((vectorcall)) c_vectorcall_check(int a, float b, double c, void *d, float e, double f, double g, float h, float i, int j) {
+ assert_or_panic(a == 1);
+ assert_or_panic(b == 2.0);
+ assert_or_panic(c == 3.0);
+ assert_or_panic((uintptr_t)d == 4);
+ assert_or_panic(e == 5.0);
+ assert_or_panic(f == 6.0);
+ assert_or_panic(g == 7.0);
+ assert_or_panic(h == 8.0);
+ assert_or_panic(i == 9.0);
+ assert_or_panic(j == 10);
+ zig_vectorcall_check(a, b, c, d, e, f, g, h, i, j);
+}
+#endif
diff --git a/test/c_abi/main.zig b/test/c_abi/main.zig
@@ -6075,7 +6075,7 @@ test "Stdcall ABI big union" {
}
extern fn c_explict_win64(ByRef) callconv(.{ .x86_64_win = .{} }) ByRef;
-test "explicit SysV calling convention" {
+test "explicit Win64 calling convention" {
if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
const res = c_explict_win64(.{ .val = 1, .arr = undefined });
@@ -6083,7 +6083,7 @@ test "explicit SysV calling convention" {
}
extern fn c_explict_sys_v(ByRef) callconv(.{ .x86_64_sysv = .{} }) ByRef;
-test "explicit Win64 calling convention" {
+test "explicit SysV calling convention" {
if (builtin.cpu.arch != .x86_64) return error.SkipZigTest;
const res = c_explict_sys_v(.{ .val = 1, .arr = undefined });
@@ -6147,3 +6147,38 @@ test "byval tail callsite attribute" {
};
try expect(v.run() == 3.0);
}
+
+test "x86 fastcall calling convention" {
+ if (builtin.cpu.arch != .x86) return error.SkipZigTest;
+ const static = struct {
+ extern fn c_fastcall_check(a: c_int, b: f32, c: *anyopaque, d: f64, e: c_int) callconv(.{ .x86_fastcall = .{} }) void;
+ export fn zig_fastcall_check(a: c_int, b: f32, c: *anyopaque, d: f64, e: c_int) callconv(.{ .x86_fastcall = .{} }) void {
+ if (a != 1) @panic("test failure");
+ if (b != 2.0) @panic("test failure");
+ if (@intFromPtr(c) != 3) @panic("test failure");
+ if (d != 4.0) @panic("test failure");
+ if (e != 5) @panic("test failure");
+ }
+ };
+ static.c_fastcall_check(1, 2.0, @ptrFromInt(3), 4.0, 5);
+}
+
+test "x86 vectorcall calling convention" {
+ if (builtin.cpu.arch != .x86) return error.SkipZigTest;
+ const static = struct {
+ extern fn c_vectorcall_check(a: c_int, b: f32, c: f64, d: *anyopaque, e: f32, f: f64, g: f64, h: f32, i: f32, j: c_int) callconv(.{ .x86_vectorcall = .{} }) void;
+ export fn zig_vectorcall_check(a: c_int, b: f32, c: f64, d: *anyopaque, e: f32, f: f64, g: f64, h: f32, i: f32, j: c_int) callconv(.{ .x86_vectorcall = .{} }) void {
+ if (a != 1) @panic("test failure");
+ if (b != 2.0) @panic("test failure");
+ if (c != 3.0) @panic("test failure");
+ if (@intFromPtr(d) != 4) @panic("test failure");
+ if (e != 5.0) @panic("test failure");
+ if (f != 6.0) @panic("test failure");
+ if (g != 7.0) @panic("test failure");
+ if (h != 8.0) @panic("test failure");
+ if (i != 9.0) @panic("test failure");
+ if (j != 10) @panic("test failure");
+ }
+ };
+ static.c_vectorcall_check(1, 2.0, 3.0, @ptrFromInt(4), 5.0, 6.0, 7.0, 8.0, 9.0, 10);
+}
