commit 45833c031def1faa53f0715202b7eb3f23d326c1 (tree)
parent fa3a9fcdfaee42ef304f662d65decbb685c705a0
Author: Matthew Lugg <mlugg@mlugg.co.uk>
Date: Sun, 14 Jun 2026 11:21:11 +0100
Air.Legalize: introduce new bitcast scalarizations
Diffstat:
4 files changed, 125 insertions(+), 39 deletions(-)
diff --git a/src/Air.zig b/src/Air.zig
@@ -955,7 +955,7 @@ pub const Inst = struct {
/// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
/// this instruction when scalarizing vector operations.
///
- /// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
+ /// Uses the `bin_op` field. `lhs` is the vector value. `rhs` is the element index. Result
/// type is the vector element type.
legalize_vec_elem_val,
diff --git a/src/Air/Legalize.zig b/src/Air/Legalize.zig
@@ -75,13 +75,6 @@ pub const Feature = enum {
scalarize_shl_sat,
scalarize_xor,
scalarize_not,
- /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements.
- /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`.
- /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast
- /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and
- /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be
- /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics.
- scalarize_bitcast,
scalarize_clz,
scalarize_ctz,
scalarize_popcount,
@@ -122,6 +115,35 @@ pub const Feature = enum {
scalarize_select,
scalarize_mul_add,
+ // Below are several different features for scalarizing `bitcast` in different scenarios. It is
+ // valid to enable any combination of these features.
+
+ /// Scalarize `bitcast` where the operand or result type is an array.
+ scalarize_bitcast_array,
+ /// Scalarize `bitcast` where either:
+ ///
+ /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
+ /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
+ ///
+ /// This effectively scalarizes any `bitcast` to/from a vector, *unless* the operation can be
+ /// performed by bitcasting each vector element and returning a vector of the results.
+ ///
+ /// If this feature is enabled, the following AIR instruction tags may be emitted:
+ /// * `.legalize_vec_elem_val`
+ /// * `.legalize_vec_store_elem`
+ scalarize_bitcast_vector_non_elementwise,
+ /// Scalarize `bitcast` where the operand or result type is an array or vector whose element
+ /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
+ /// need to sign- or zero-extend multiple elements to populate "padding" bits.
+ ///
+ /// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig`
+ /// to conform to the new bitcast semantics.
+ ///
+ /// If this feature is enabled, the following AIR instruction tags may be emitted:
+ /// * `.legalize_vec_elem_val`
+ /// * `.legalize_vec_store_elem`
+ scalarize_bitcast_padded_elems,
+
/// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
unsplat_shift_rhs,
/// Legalize reduce of a one element vector to a bitcast.
@@ -227,7 +249,6 @@ pub const Feature = enum {
.shl_sat => .scalarize_shl_sat,
.xor => .scalarize_xor,
.not => .scalarize_not,
- .bitcast => .scalarize_bitcast,
.clz => .scalarize_clz,
.ctz => .scalarize_ctz,
.popcount => .scalarize_popcount,
@@ -548,7 +569,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
},
}
},
- .bitcast => if (l.features.has(.scalarize_bitcast)) {
+ .bitcast => if (l.features.hasAny(&.{
+ .scalarize_bitcast_array,
+ .scalarize_bitcast_vector_non_elementwise,
+ .scalarize_bitcast_padded_elems,
+ })) {
if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
continue :inst l.replaceInst(inst, .block, payload);
}
@@ -1423,35 +1448,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
const dest_ty = ty_op.ty.toType();
- const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
- else => true,
- .array, .vector => legal: {
- if (dest_ty.arrayLen(zcu) == 1) break :legal true;
- const dest_elem_ty = dest_ty.childType(zcu);
- break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
- },
- };
-
const operand_ty = l.typeOf(ty_op.operand);
- const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
- else => true,
- .array, .vector => legal: {
- if (operand_ty.arrayLen(zcu) == 1) break :legal true;
- const operand_elem_ty = operand_ty.childType(zcu);
- break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
- },
- };
- if (dest_legal and operand_legal) return null;
+ // We exit this block only if the scalarization is actually necessary. Otherwise we will return
+ // `null` from within the block.
+ const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: {
+ const operand_tag = operand_ty.zigTypeTag(zcu);
+ const dest_tag = dest_ty.zigTypeTag(zcu);
+
+ if (operand_tag != .array and
+ operand_tag != .vector and
+ dest_tag != .array and
+ dest_tag != .vector)
+ {
+ return null;
+ }
- if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
- // from_ty and to_ty are both arrays or vectors of types with the same bit size,
- // so we can do an elementwise bitcast.
- return try l.scalarizeBlockPayload(orig_inst, .ty_op);
- }
+ // We track the validity of 3 different bitcast operations:
+ // * operand -> dest
+ // * operand -> uint
+ // * uint -> dest
+ // If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing
+ // the validity of the other operations helps us lower the scalarization efficiently.
+ var operand_to_dest: bool = true;
+ var operand_to_int: bool = true;
+ var int_to_dest: bool = true;
+
+ if (l.features.has(.scalarize_bitcast_array)) {
+ if (operand_tag == .array) {
+ operand_to_dest = false;
+ operand_to_int = false;
+ }
+ if (dest_tag == .array) {
+ operand_to_dest = false;
+ int_to_dest = false;
+ }
+ }
+
+ if (l.features.has(.scalarize_bitcast_vector_non_elementwise)) {
+ if (operand_tag == .vector) operand_to_int = false;
+ if (dest_tag == .vector) int_to_dest = false;
+
+ if (operand_tag == .vector or dest_tag == .vector) {
+ if (operand_tag != .vector or
+ dest_tag != .vector or
+ operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu))
+ {
+ operand_to_dest = false;
+ }
+ }
+ }
+
+ if (l.features.has(.scalarize_bitcast_padded_elems)) {
+ if (operand_tag == .array or operand_tag == .vector) {
+ const elem_ty = operand_ty.childType(zcu);
+ if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
+ operand_to_int = false;
+ operand_to_dest = false;
+ }
+ }
+ if (dest_tag == .array or dest_tag == .vector) {
+ const elem_ty = dest_ty.childType(zcu);
+ if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
+ int_to_dest = false;
+ operand_to_dest = false;
+ }
+ }
+ }
+
+ if (operand_to_dest) {
+ return null; // no scalarization needed!
+ }
+
+ // We need a scalarization, but before breaking from the block, check if we can do it
+ // elementwise---if we can, that's preferable to the generic lowering.
+ if ((operand_tag == .array or operand_tag == .vector) and
+ (dest_tag == .array or dest_tag == .vector) and
+ operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu))
+ {
+ // Operand and result types are both arrays/vectors whose element types have the same
+ // bit size, so we can do an elementwise bitcast.
+ return try l.scalarizeBlockPayload(orig_inst, .ty_op);
+ }
+
+ break :int_ok .{ operand_to_int, int_to_dest };
+ };
- // Fallback path. Our strategy is to use an unsigned integer type as an intermediate
- // "bag of bits" representation which can be manipulated by bitwise operations.
+ // Generic scalarization implementation. Our strategy is to use an unsigned integer type as an
+ // intermediate "bag of bits" representation which can be manipulated by bitwise operations.
const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
assert(operand_ty.bitSize(zcu) == num_bits);
@@ -1465,7 +1549,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
// First, convert `operand_ty` to `uint_ty` (`uN`).
const uint_val: Air.Inst.Ref = uint_val: {
- if (operand_legal) {
+ if (operand_to_int_ok) {
_ = main_block.stealCapacity(19);
break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
}
@@ -1560,7 +1644,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
// Now convert `uint_ty` (`uN`) to `dest_ty`.
- if (dest_legal) {
+ if (int_to_dest_ok) {
_ = main_block.stealCapacity(17);
const result = main_block.addBitCast(l, dest_ty, uint_val);
main_block.addBr(l, orig_inst, result);
diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig
@@ -83,7 +83,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
.scalarize_shl_sat,
.scalarize_xor,
.scalarize_not,
- .scalarize_bitcast,
.scalarize_clz,
.scalarize_ctz,
.scalarize_popcount,
@@ -120,6 +119,8 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
.scalarize_shuffle_two,
.scalarize_select,
.scalarize_mul_add,
+
+ .scalarize_bitcast_padded_elems,
});
}
diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig
@@ -47,7 +47,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
.scalarize_shl,
.scalarize_shl_exact,
.scalarize_shl_sat,
- .scalarize_bitcast,
.scalarize_ctz,
.scalarize_popcount,
.scalarize_byte_swap,
@@ -58,6 +57,8 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
.scalarize_shuffle_two,
.scalarize_select,
+ .scalarize_bitcast_padded_elems,
+
//.unsplat_shift_rhs,
.reduce_one_elem_to_bitcast,
.splat_one_elem_to_bitcast,
