From 806097c165ce26d0a1817a0f2189b20ad5ef796d Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 20 Sep 2020 18:54:23 +0200 Subject: [PATCH 1/4] std: Make C allocator respect the required alignment Use posix_memalign where available and the _aligned_{malloc,free} API on Windows. Closes #3783 --- lib/std/c.zig | 3 +- lib/std/c/windows.zig | 4 + lib/std/heap.zig | 230 ++++++++++++++++++++++++++---------------- 3 files changed, 150 insertions(+), 87 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 7d2d200e7e..5a26b6f9f6 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -245,7 +245,6 @@ pub extern "c" fn setregid(rgid: gid_t, egid: gid_t) c_int; pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int; pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int; -pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?*c_void; pub extern "c" fn malloc(usize) ?*c_void; pub usingnamespace switch (builtin.os.tag) { @@ -260,7 +259,7 @@ pub usingnamespace switch (builtin.os.tag) { pub extern "c" fn realloc(?*c_void, usize) ?*c_void; pub extern "c" fn free(*c_void) void; -pub extern "c" fn posix_memalign(memptr: **c_void, alignment: usize, size: usize) c_int; +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; pub extern "c" fn futimes(fd: fd_t, times: *[2]timeval) c_int; pub extern "c" fn utimes(path: [*:0]const u8, times: *[2]timeval) c_int; diff --git a/lib/std/c/windows.zig b/lib/std/c/windows.zig index cd39324192..0c6019a218 100644 --- a/lib/std/c/windows.zig +++ b/lib/std/c/windows.zig @@ -4,3 +4,7 @@ // The MIT license requires this copyright notice to be included in all copies // and substantial portions of the software. pub extern "c" fn _errno() *c_int; + +pub extern "c" fn _aligned_free(memblock: ?*c_void) void; +pub extern "c" fn _aligned_malloc(size: usize, alignment: usize) ?*c_void; +pub extern "c" fn _aligned_realloc(memblock: ?*c_void, size: usize, alignment: usize) ?*c_void; diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 3e74db386e..ef465fbae8 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -21,68 +21,120 @@ pub const GeneralPurposeAllocator = @import("heap/general_purpose_allocator.zig" const Allocator = mem.Allocator; -usingnamespace if (comptime @hasDecl(c, "malloc_size")) - struct { - pub const supports_malloc_size = true; - pub const malloc_size = c.malloc_size; +const CAllocator = struct { + comptime { + if (!builtin.link_libc) { + @compileError("C allocator is only available when linking against libc"); + } } -else if (comptime @hasDecl(c, "malloc_usable_size")) - struct { - pub const supports_malloc_size = true; - pub const malloc_size = c.malloc_usable_size; + + usingnamespace if (comptime @hasDecl(c, "malloc_size")) + struct { + pub const supports_malloc_size = true; + pub const malloc_size = c.malloc_size; + } + else if (comptime @hasDecl(c, "malloc_usable_size")) + struct { + pub const supports_malloc_size = true; + pub const malloc_size = c.malloc_usable_size; + } + else + struct { + pub const supports_malloc_size = false; + }; + + // The alignment guaranteed by malloc, the value matches the result of the C + // expression `alignof(max_alignment_t)` + const min_ptr_alignment = comptime std.math.max( + @alignOf(c_longdouble), + @alignOf(c_longlong), + ); + + fn aligned_alloc(len: usize, alignment: usize) ?*c_void { + // The minimum alignment supported by both APIs is the size of a pointer + const eff_alignment = std.math.max(alignment, @sizeOf(usize)); + + if (builtin.os.tag == .windows) { + return c._aligned_malloc(len, eff_alignment); + } + + var aligned_ptr: ?*c_void = undefined; + if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0) + return null; + return aligned_ptr; } -else - struct { - pub const supports_malloc_size = false; - }; + + fn aligned_free(ptr: *c_void) void { + if (builtin.os.tag == .windows) { + return c._aligned_free(ptr); + } + c.free(ptr); + } + + fn alloc( + allocator: *Allocator, + len: usize, + alignment: u29, + len_align: u29, + return_address: usize, + ) error{OutOfMemory}![]u8 { + assert(len > 0); + assert(std.math.isPowerOfTwo(alignment)); + + var ptr = if (alignment <= min_ptr_alignment) + @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory) + else + @ptrCast([*]u8, aligned_alloc(len, alignment) orelse return error.OutOfMemory); + + if (len_align == 0) + return ptr[0..len]; + + const full_len = init: { + if (supports_malloc_size) { + const s = malloc_size(ptr); + assert(s >= len); + break :init s; + } + break :init len; + }; + + return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)]; + } + + fn resize( + allocator: *Allocator, + buf: []u8, + buf_align: u29, + new_len: usize, + len_align: u29, + return_address: usize, + ) Allocator.Error!usize { + if (new_len == 0) { + if (buf_align <= min_ptr_alignment) + c.free(buf.ptr) + else + aligned_free(buf.ptr); + return 0; + } + if (new_len <= buf.len) { + return mem.alignAllocLen(buf.len, new_len, len_align); + } + if (supports_malloc_size) { + const full_len = malloc_size(buf.ptr); + if (new_len <= full_len) { + return mem.alignAllocLen(full_len, new_len, len_align); + } + } + return error.OutOfMemory; + } +}; pub const c_allocator = &c_allocator_state; var c_allocator_state = Allocator{ - .allocFn = cAlloc, - .resizeFn = cResize, + .allocFn = CAllocator.alloc, + .resizeFn = CAllocator.resize, }; -fn cAlloc(self: *Allocator, len: usize, ptr_align: u29, len_align: u29, ret_addr: usize) Allocator.Error![]u8 { - assert(ptr_align <= @alignOf(c_longdouble)); - const ptr = @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory); - if (len_align == 0) { - return ptr[0..len]; - } - const full_len = init: { - if (supports_malloc_size) { - const s = malloc_size(ptr); - assert(s >= len); - break :init s; - } - break :init len; - }; - return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)]; -} - -fn cResize( - self: *Allocator, - buf: []u8, - old_align: u29, - new_len: usize, - len_align: u29, - ret_addr: usize, -) Allocator.Error!usize { - if (new_len == 0) { - c.free(buf.ptr); - return 0; - } - if (new_len <= buf.len) { - return mem.alignAllocLen(buf.len, new_len, len_align); - } - if (supports_malloc_size) { - const full_len = malloc_size(buf.ptr); - if (new_len <= full_len) { - return mem.alignAllocLen(full_len, new_len, len_align); - } - } - return error.OutOfMemory; -} - /// This allocator makes a syscall directly for every allocation and free. /// Thread-safe and lock-free. pub const page_allocator = if (std.Target.current.isWasm()) @@ -726,9 +778,10 @@ pub fn StackFallbackAllocator(comptime size: usize) type { test "c_allocator" { if (builtin.link_libc) { - var slice = try c_allocator.alloc(u8, 50); - defer c_allocator.free(slice); - slice = try c_allocator.realloc(slice, 100); + try testAllocator(c_allocator); + try testAllocatorAligned(c_allocator); + try testAllocatorLargeAlignment(c_allocator); + try testAllocatorAlignedShrink(c_allocator); } } @@ -772,7 +825,7 @@ test "WasmPageAllocator internals" { test "PageAllocator" { const allocator = page_allocator; try testAllocator(allocator); - try testAllocatorAligned(allocator, 16); + try testAllocatorAligned(allocator); if (!std.Target.current.isWasm()) { try testAllocatorLargeAlignment(allocator); try testAllocatorAlignedShrink(allocator); @@ -802,7 +855,7 @@ test "HeapAllocator" { const allocator = &heap_allocator.allocator; try testAllocator(allocator); - try testAllocatorAligned(allocator, 16); + try testAllocatorAligned(allocator); try testAllocatorLargeAlignment(allocator); try testAllocatorAlignedShrink(allocator); } @@ -813,7 +866,7 @@ test "ArenaAllocator" { defer arena_allocator.deinit(); try testAllocator(&arena_allocator.allocator); - try testAllocatorAligned(&arena_allocator.allocator, 16); + try testAllocatorAligned(&arena_allocator.allocator); try testAllocatorLargeAlignment(&arena_allocator.allocator); try testAllocatorAlignedShrink(&arena_allocator.allocator); } @@ -823,7 +876,7 @@ test "FixedBufferAllocator" { var fixed_buffer_allocator = mem.validationWrap(FixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..])); try testAllocator(&fixed_buffer_allocator.allocator); - try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16); + try testAllocatorAligned(&fixed_buffer_allocator.allocator); try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator); try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator); } @@ -881,7 +934,7 @@ test "ThreadSafeFixedBufferAllocator" { var fixed_buffer_allocator = ThreadSafeFixedBufferAllocator.init(test_fixed_buffer_allocator_memory[0..]); try testAllocator(&fixed_buffer_allocator.allocator); - try testAllocatorAligned(&fixed_buffer_allocator.allocator, 16); + try testAllocatorAligned(&fixed_buffer_allocator.allocator); try testAllocatorLargeAlignment(&fixed_buffer_allocator.allocator); try testAllocatorAlignedShrink(&fixed_buffer_allocator.allocator); } @@ -916,6 +969,10 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { allocator.free(slice); + // Zero-length allocation + var empty = try allocator.alloc(u8, 0); + allocator.free(empty); + // Allocation with zero-sized types const zero_bit_ptr = try allocator.create(u0); zero_bit_ptr.* = 0; allocator.destroy(zero_bit_ptr); @@ -928,31 +985,34 @@ pub fn testAllocator(base_allocator: *mem.Allocator) !void { allocator.free(oversize); } -pub fn testAllocatorAligned(base_allocator: *mem.Allocator, comptime alignment: u29) !void { +pub fn testAllocatorAligned(base_allocator: *mem.Allocator) !void { var validationAllocator = mem.validationWrap(base_allocator); const allocator = &validationAllocator.allocator; - // initial - var slice = try allocator.alignedAlloc(u8, alignment, 10); - testing.expect(slice.len == 10); - // grow - slice = try allocator.realloc(slice, 100); - testing.expect(slice.len == 100); - // shrink - slice = allocator.shrink(slice, 10); - testing.expect(slice.len == 10); - // go to zero - slice = allocator.shrink(slice, 0); - testing.expect(slice.len == 0); - // realloc from zero - slice = try allocator.realloc(slice, 100); - testing.expect(slice.len == 100); - // shrink with shrink - slice = allocator.shrink(slice, 10); - testing.expect(slice.len == 10); - // shrink to zero - slice = allocator.shrink(slice, 0); - testing.expect(slice.len == 0); + // Test a few alignment values, smaller and bigger than the type's one + inline for ([_]u29{ 1, 2, 4, 8, 16, 32, 64 }) |alignment| { + // initial + var slice = try allocator.alignedAlloc(u8, alignment, 10); + testing.expect(slice.len == 10); + // grow + slice = try allocator.realloc(slice, 100); + testing.expect(slice.len == 100); + // shrink + slice = allocator.shrink(slice, 10); + testing.expect(slice.len == 10); + // go to zero + slice = allocator.shrink(slice, 0); + testing.expect(slice.len == 0); + // realloc from zero + slice = try allocator.realloc(slice, 100); + testing.expect(slice.len == 100); + // shrink with shrink + slice = allocator.shrink(slice, 10); + testing.expect(slice.len == 10); + // shrink to zero + slice = allocator.shrink(slice, 0); + testing.expect(slice.len == 0); + } } pub fn testAllocatorLargeAlignment(base_allocator: *mem.Allocator) mem.Allocator.Error!void { From 4664eae1e40813fb50bab83247cc7391312e38cb Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Mon, 21 Sep 2020 23:29:34 +0200 Subject: [PATCH 2/4] stage1: Fix type mapping for c_longdouble A quick and dirty job to let the compiler use the correct size and alignment. --- src/stage1/codegen.cpp | 52 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp index c034a79cea..dae5997f0e 100644 --- a/src/stage1/codegen.cpp +++ b/src/stage1/codegen.cpp @@ -8485,7 +8485,57 @@ static void define_builtin_types(CodeGen *g) { add_fp_entry(g, "f32", 32, LLVMFloatType(), &g->builtin_types.entry_f32); add_fp_entry(g, "f64", 64, LLVMDoubleType(), &g->builtin_types.entry_f64); add_fp_entry(g, "f128", 128, LLVMFP128Type(), &g->builtin_types.entry_f128); - add_fp_entry(g, "c_longdouble", 80, LLVMX86FP80Type(), &g->builtin_types.entry_c_longdouble); + + switch (g->zig_target->arch) { + case ZigLLVM_x86: + case ZigLLVM_x86_64: + if (g->zig_target->abi != ZigLLVM_MSVC) + add_fp_entry(g, "c_longdouble", 80, LLVMX86FP80Type(), &g->builtin_types.entry_c_longdouble); + else + add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_arm: + case ZigLLVM_armeb: + case ZigLLVM_thumb: + case ZigLLVM_thumbeb: + add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_aarch64: + case ZigLLVM_aarch64_be: + if (g->zig_target->os == OsWindows || target_os_is_darwin(g->zig_target->os)) + add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble); + else + add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_riscv32: + case ZigLLVM_riscv64: + add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_wasm32: + case ZigLLVM_wasm64: + add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_mips: + case ZigLLVM_mipsel: + // Assume o32 ABI + add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_mips64: + case ZigLLVM_mips64el: + add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_ppc: + case ZigLLVM_ppc64: + case ZigLLVM_ppc64le: + add_fp_entry(g, "c_longdouble", 128, LLVMFP128Type(), &g->builtin_types.entry_c_longdouble); + break; + case ZigLLVM_avr: + // It's either a float or a double, depending on a toolchain switch + add_fp_entry(g, "c_longdouble", 64, LLVMDoubleType(), &g->builtin_types.entry_c_longdouble); + break; + default: + zig_panic("TODO implement mapping for c_longdouble"); + } { ZigType *entry = new_type_table_entry(ZigTypeIdVoid); From 53433cdea2cee73caea52a6baa10dc7bc7f6da4d Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Sep 2020 13:18:40 +0200 Subject: [PATCH 3/4] Implement a fallback mechanism for posix_memalign Do the alignment dance by ourselves whenever posix_memalign is not available. Don't try to use malloc as it has too many edge cases, figuring out whether a block of memory is manually aligned by the mechanism above or is directly coming from malloc becomes too hard to be valuable. --- lib/std/c.zig | 12 ------ lib/std/c/darwin.zig | 3 ++ lib/std/c/dragonfly.zig | 2 + lib/std/c/freebsd.zig | 3 ++ lib/std/c/linux.zig | 2 + lib/std/c/netbsd.zig | 3 ++ lib/std/c/openbsd.zig | 3 ++ lib/std/c/windows.zig | 4 +- lib/std/heap.zig | 82 +++++++++++++++++++++++++---------------- 9 files changed, 67 insertions(+), 47 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 5a26b6f9f6..4b143d499f 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -246,20 +246,8 @@ pub extern "c" fn setresuid(ruid: uid_t, euid: uid_t, suid: uid_t) c_int; pub extern "c" fn setresgid(rgid: gid_t, egid: gid_t, sgid: gid_t) c_int; pub extern "c" fn malloc(usize) ?*c_void; - -pub usingnamespace switch (builtin.os.tag) { - .linux, .freebsd, .kfreebsd, .netbsd => struct { - pub extern "c" fn malloc_usable_size(?*const c_void) usize; - }, - .macos, .ios, .watchos, .tvos => struct { - pub extern "c" fn malloc_size(?*const c_void) usize; - }, - else => struct {}, -}; - pub extern "c" fn realloc(?*c_void, usize) ?*c_void; pub extern "c" fn free(*c_void) void; -pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; pub extern "c" fn futimes(fd: fd_t, times: *[2]timeval) c_int; pub extern "c" fn utimes(path: [*:0]const u8, times: *[2]timeval) c_int; diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig index e0acd6c746..86a2bf5ee4 100644 --- a/lib/std/c/darwin.zig +++ b/lib/std/c/darwin.zig @@ -45,6 +45,9 @@ pub const _fstatat = if (builtin.arch == .aarch64) fstatat else @"fstatat$INODE6 pub extern "c" fn mach_absolute_time() u64; pub extern "c" fn mach_timebase_info(tinfo: ?*mach_timebase_info_data) void; +pub extern "c" fn malloc_size(?*const c_void) usize; +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; + pub extern "c" fn kevent64( kq: c_int, changelist: [*]const kevent64_s, diff --git a/lib/std/c/dragonfly.zig b/lib/std/c/dragonfly.zig index 2550cacc5b..3261d34b78 100644 --- a/lib/std/c/dragonfly.zig +++ b/lib/std/c/dragonfly.zig @@ -17,6 +17,8 @@ pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize pub const dl_iterate_phdr_callback = fn (info: *dl_phdr_info, size: usize, data: ?*c_void) callconv(.C) c_int; pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int; +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; + pub const pthread_mutex_t = extern struct { inner: ?*c_void = null, }; diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig index 1545a0e153..8fa78b0d6f 100644 --- a/lib/std/c/freebsd.zig +++ b/lib/std/c/freebsd.zig @@ -13,6 +13,9 @@ pub extern "c" fn getdents(fd: c_int, buf_ptr: [*]u8, nbytes: usize) usize; pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int; pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize; +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; +pub extern "c" fn malloc_usable_size(?*const c_void) usize; + pub const sf_hdtr = extern struct { headers: [*]const iovec_const, hdr_cnt: c_int, diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index d48c76c7a6..947a92a1e4 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -101,6 +101,8 @@ pub extern "c" fn copy_file_range(fd_in: fd_t, off_in: ?*i64, fd_out: fd_t, off_ pub extern "c" fn signalfd(fd: fd_t, mask: *const sigset_t, flags: c_uint) c_int; pub extern "c" fn prlimit(pid: pid_t, resource: rlimit_resource, new_limit: *const rlimit, old_limit: *rlimit) c_int; +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; +pub extern "c" fn malloc_usable_size(?*const c_void) usize; pub const pthread_attr_t = extern struct { __size: [56]u8, diff --git a/lib/std/c/netbsd.zig b/lib/std/c/netbsd.zig index 0eb8e32bae..e6d7e86bee 100644 --- a/lib/std/c/netbsd.zig +++ b/lib/std/c/netbsd.zig @@ -30,6 +30,9 @@ pub extern "c" fn __getrusage50(who: c_int, usage: *rusage) c_int; // libc aliases this as sched_yield pub extern "c" fn __libc_thr_yield() c_int; +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; +pub extern "c" fn malloc_usable_size(?*const c_void) usize; + pub const pthread_mutex_t = extern struct { ptm_magic: u32 = 0x33330003, ptm_errorcheck: padded_pthread_spin_t = 0, diff --git a/lib/std/c/openbsd.zig b/lib/std/c/openbsd.zig index f31ab71eee..48feaff428 100644 --- a/lib/std/c/openbsd.zig +++ b/lib/std/c/openbsd.zig @@ -32,3 +32,6 @@ pub const pthread_spinlock_t = extern struct { pub const pthread_attr_t = extern struct { inner: ?*c_void = null, }; + +pub extern "c" fn posix_memalign(memptr: *?*c_void, alignment: usize, size: usize) c_int; +pub extern "c" fn malloc_usable_size(?*const c_void) usize; diff --git a/lib/std/c/windows.zig b/lib/std/c/windows.zig index 0c6019a218..f96da56c1f 100644 --- a/lib/std/c/windows.zig +++ b/lib/std/c/windows.zig @@ -5,6 +5,4 @@ // and substantial portions of the software. pub extern "c" fn _errno() *c_int; -pub extern "c" fn _aligned_free(memblock: ?*c_void) void; -pub extern "c" fn _aligned_malloc(size: usize, alignment: usize) ?*c_void; -pub extern "c" fn _aligned_realloc(memblock: ?*c_void, size: usize, alignment: usize) ?*c_void; +pub extern "c" fn _msize(memblock: ?*c_void) usize; diff --git a/lib/std/heap.zig b/lib/std/heap.zig index ef465fbae8..431e93d6c2 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -38,37 +38,63 @@ const CAllocator = struct { pub const supports_malloc_size = true; pub const malloc_size = c.malloc_usable_size; } + else if (comptime @hasDecl(c, "_msize")) + struct { + pub const supports_malloc_size = true; + pub const malloc_size = c._msize; + } else struct { pub const supports_malloc_size = false; }; - // The alignment guaranteed by malloc, the value matches the result of the C - // expression `alignof(max_alignment_t)` - const min_ptr_alignment = comptime std.math.max( - @alignOf(c_longdouble), - @alignOf(c_longlong), - ); + pub const supports_posix_memalign = false and @hasDecl(c, "posix_memalign"); - fn aligned_alloc(len: usize, alignment: usize) ?*c_void { - // The minimum alignment supported by both APIs is the size of a pointer - const eff_alignment = std.math.max(alignment, @sizeOf(usize)); + fn get_header(ptr: [*]u8) *[*]u8 { + return @intToPtr(*[*]u8, @ptrToInt(ptr) - @sizeOf(usize)); + } - if (builtin.os.tag == .windows) { - return c._aligned_malloc(len, eff_alignment); + fn aligned_alloc(len: usize, alignment: usize) ?[*]u8 { + if (supports_posix_memalign) { + // The minimum alignment supported posix_memalign is the pointer size + const eff_alignment = std.math.max(alignment, @sizeOf(usize)); + + var aligned_ptr: ?*c_void = undefined; + if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0) + return null; + + return @ptrCast([*]u8, aligned_ptr); } - var aligned_ptr: ?*c_void = undefined; - if (c.posix_memalign(&aligned_ptr, eff_alignment, len) != 0) - return null; + // Thin wrapper around regular malloc, overallocate to account for + // alignment padding and store the orignal malloc()'ed pointer before + // the aligned address. + var unaligned_ptr = @ptrCast([*]u8, c.malloc(len + alignment - 1 + @sizeOf(usize)) orelse return null); + const unaligned_addr = @ptrToInt(unaligned_ptr); + const aligned_addr = mem.alignForward(unaligned_addr + @sizeOf(usize), alignment); + var aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr); + get_header(aligned_ptr).* = unaligned_ptr; + return aligned_ptr; } - fn aligned_free(ptr: *c_void) void { - if (builtin.os.tag == .windows) { - return c._aligned_free(ptr); + fn aligned_free(ptr: [*]u8) void { + if (supports_posix_memalign) { + return c.free(ptr); } - c.free(ptr); + + const unaligned_ptr = get_header(ptr).*; + c.free(unaligned_ptr); + } + + fn aligned_alloc_size(ptr: [*]u8) usize { + if (supports_posix_memalign) { + return malloc_size(ptr); + } + + const unaligned_ptr = get_header(ptr).*; + const delta = @ptrToInt(ptr) - @ptrToInt(unaligned_ptr); + return malloc_size(unaligned_ptr) - delta; } fn alloc( @@ -81,23 +107,18 @@ const CAllocator = struct { assert(len > 0); assert(std.math.isPowerOfTwo(alignment)); - var ptr = if (alignment <= min_ptr_alignment) - @ptrCast([*]u8, c.malloc(len) orelse return error.OutOfMemory) - else - @ptrCast([*]u8, aligned_alloc(len, alignment) orelse return error.OutOfMemory); - - if (len_align == 0) + var ptr = aligned_alloc(len, alignment) orelse return error.OutOfMemory; + if (len_align == 0) { return ptr[0..len]; - + } const full_len = init: { if (supports_malloc_size) { - const s = malloc_size(ptr); + const s = aligned_alloc_size(ptr); assert(s >= len); break :init s; } break :init len; }; - return ptr[0..mem.alignBackwardAnyAlign(full_len, len_align)]; } @@ -110,17 +131,14 @@ const CAllocator = struct { return_address: usize, ) Allocator.Error!usize { if (new_len == 0) { - if (buf_align <= min_ptr_alignment) - c.free(buf.ptr) - else - aligned_free(buf.ptr); + aligned_free(buf.ptr); return 0; } if (new_len <= buf.len) { return mem.alignAllocLen(buf.len, new_len, len_align); } if (supports_malloc_size) { - const full_len = malloc_size(buf.ptr); + const full_len = aligned_alloc_size(buf.ptr); if (new_len <= full_len) { return mem.alignAllocLen(full_len, new_len, len_align); } From e749ab1d63cfdacb3ac9e511d1ef52e77799c82b Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 22 Sep 2020 14:42:01 +0200 Subject: [PATCH 4/4] Fix typo, remove debug leftover, rename few fns --- lib/std/heap.zig | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 431e93d6c2..4db08a8be0 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -48,15 +48,16 @@ const CAllocator = struct { pub const supports_malloc_size = false; }; - pub const supports_posix_memalign = false and @hasDecl(c, "posix_memalign"); + pub const supports_posix_memalign = @hasDecl(c, "posix_memalign"); - fn get_header(ptr: [*]u8) *[*]u8 { + fn getHeader(ptr: [*]u8) *[*]u8 { return @intToPtr(*[*]u8, @ptrToInt(ptr) - @sizeOf(usize)); } - fn aligned_alloc(len: usize, alignment: usize) ?[*]u8 { + fn alignedAlloc(len: usize, alignment: usize) ?[*]u8 { if (supports_posix_memalign) { - // The minimum alignment supported posix_memalign is the pointer size + // The posix_memalign only accepts alignment values that are a + // multiple of the pointer size const eff_alignment = std.math.max(alignment, @sizeOf(usize)); var aligned_ptr: ?*c_void = undefined; @@ -73,26 +74,26 @@ const CAllocator = struct { const unaligned_addr = @ptrToInt(unaligned_ptr); const aligned_addr = mem.alignForward(unaligned_addr + @sizeOf(usize), alignment); var aligned_ptr = unaligned_ptr + (aligned_addr - unaligned_addr); - get_header(aligned_ptr).* = unaligned_ptr; + getHeader(aligned_ptr).* = unaligned_ptr; return aligned_ptr; } - fn aligned_free(ptr: [*]u8) void { + fn alignedFree(ptr: [*]u8) void { if (supports_posix_memalign) { return c.free(ptr); } - const unaligned_ptr = get_header(ptr).*; + const unaligned_ptr = getHeader(ptr).*; c.free(unaligned_ptr); } - fn aligned_alloc_size(ptr: [*]u8) usize { + fn alignedAllocSize(ptr: [*]u8) usize { if (supports_posix_memalign) { return malloc_size(ptr); } - const unaligned_ptr = get_header(ptr).*; + const unaligned_ptr = getHeader(ptr).*; const delta = @ptrToInt(ptr) - @ptrToInt(unaligned_ptr); return malloc_size(unaligned_ptr) - delta; } @@ -107,13 +108,13 @@ const CAllocator = struct { assert(len > 0); assert(std.math.isPowerOfTwo(alignment)); - var ptr = aligned_alloc(len, alignment) orelse return error.OutOfMemory; + var ptr = alignedAlloc(len, alignment) orelse return error.OutOfMemory; if (len_align == 0) { return ptr[0..len]; } const full_len = init: { if (supports_malloc_size) { - const s = aligned_alloc_size(ptr); + const s = alignedAllocSize(ptr); assert(s >= len); break :init s; } @@ -131,14 +132,14 @@ const CAllocator = struct { return_address: usize, ) Allocator.Error!usize { if (new_len == 0) { - aligned_free(buf.ptr); + alignedFree(buf.ptr); return 0; } if (new_len <= buf.len) { return mem.alignAllocLen(buf.len, new_len, len_align); } if (supports_malloc_size) { - const full_len = aligned_alloc_size(buf.ptr); + const full_len = alignedAllocSize(buf.ptr); if (new_len <= full_len) { return mem.alignAllocLen(full_len, new_len, len_align); }