diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 3384374dd6..c4923127ee 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -8,7 +8,7 @@ const Writer = std.io.Writer; const assert = std.debug.assert; const testing = std.testing; const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayListUnmanaged; +const ArrayList = std.ArrayList; const Limit = std.io.Limit; pub const Limited = @import("Reader/Limited.zig"); @@ -290,103 +290,63 @@ pub const LimitedAllocError = Allocator.Error || ShortError || error{StreamTooLo pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocError![]u8 { var buffer: ArrayList(u8) = .empty; defer buffer.deinit(gpa); - try appendRemaining(r, gpa, null, &buffer, limit); + try appendRemaining(r, gpa, &buffer, limit); return buffer.toOwnedSlice(gpa); } /// Transfers all bytes from the current position to the end of the stream, up /// to `limit`, appending them to `list`. /// -/// If `limit` would be exceeded, `error.StreamTooLong` is returned instead. In -/// such case, the next byte that would be read will be the first one to exceed -/// `limit`, and all preceeding bytes have been appended to `list`. -/// -/// If `limit` is not `Limit.unlimited`, asserts `buffer` has nonzero capacity. +/// If `limit` is reached or exceeded, `error.StreamTooLong` is returned +/// instead. In such case, the next byte that would be read will be the first +/// one to exceed `limit`, and all preceeding bytes have been appended to +/// `list`. /// /// See also: /// * `allocRemaining` pub fn appendRemaining( r: *Reader, gpa: Allocator, - comptime alignment: ?std.mem.Alignment, - list: *std.ArrayListAlignedUnmanaged(u8, alignment), + list: *ArrayList(u8), limit: Limit, ) LimitedAllocError!void { - if (limit == .unlimited) return appendRemainingUnlimited(r, gpa, alignment, list, 1); - assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data. - const buffer_contents = r.buffer[r.seek..r.end]; - const copy_len = limit.minInt(buffer_contents.len); - try list.appendSlice(gpa, r.buffer[0..copy_len]); - r.seek += copy_len; - if (buffer_contents.len - copy_len != 0) return error.StreamTooLong; - r.seek = 0; - r.end = 0; - var remaining = @intFromEnum(limit) - copy_len; - // From here, we leave `buffer` empty, appending directly to `list`. - var writer: Writer = .{ - .buffer = undefined, - .end = undefined, - .vtable = &.{ .drain = Writer.fixedDrain }, - }; - while (true) { - try list.ensureUnusedCapacity(gpa, 2); - const cap = list.unusedCapacitySlice(); - const dest = cap[0..@min(cap.len, remaining + 1)]; - writer.buffer = list.allocatedSlice(); - writer.end = list.items.len; - const n = r.vtable.stream(r, &writer, .limited(dest.len)) catch |err| switch (err) { - error.WriteFailed => unreachable, // Prevented by the limit. + var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.items); + a.writer.end = list.items.len; + list.* = .empty; + defer { + list.* = .{ + .items = a.writer.buffer[0..a.writer.end], + .capacity = a.writer.buffer.len, + }; + } + var remaining = limit; + while (remaining.nonzero()) { + const n = stream(r, &a.writer, remaining) catch |err| switch (err) { error.EndOfStream => return, + error.WriteFailed => return error.OutOfMemory, error.ReadFailed => return error.ReadFailed, }; - list.items.len += n; - if (n > remaining) { - // Move the byte to `Reader.buffer` so it is not lost. - assert(n - remaining == 1); - assert(r.end == 0); - r.buffer[0] = list.items[list.items.len - 1]; - list.items.len -= 1; - r.end = 1; - return; - } - remaining -= n; + remaining = remaining.subtract(n).?; } + return error.StreamTooLong; } pub const UnlimitedAllocError = Allocator.Error || ShortError; -pub fn appendRemainingUnlimited( - r: *Reader, - gpa: Allocator, - comptime alignment: ?std.mem.Alignment, - list: *std.ArrayListAlignedUnmanaged(u8, alignment), - bump: usize, -) UnlimitedAllocError!void { - const buffer_contents = r.buffer[r.seek..r.end]; - try list.ensureUnusedCapacity(gpa, buffer_contents.len + bump); - list.appendSliceAssumeCapacity(buffer_contents); - // If statement protects `ending`. - if (r.end != 0) { - r.seek = 0; - r.end = 0; - } - // From here, we leave `buffer` empty, appending directly to `list`. - var writer: Writer = .{ - .buffer = undefined, - .end = undefined, - .vtable = &.{ .drain = Writer.fixedDrain }, - }; - while (true) { - try list.ensureUnusedCapacity(gpa, bump); - writer.buffer = list.allocatedSlice(); - writer.end = list.items.len; - const n = r.vtable.stream(r, &writer, .limited(list.unusedCapacitySlice().len)) catch |err| switch (err) { - error.WriteFailed => unreachable, // Prevented by the limit. - error.EndOfStream => return, - error.ReadFailed => return error.ReadFailed, +pub fn appendRemainingUnlimited(r: *Reader, gpa: Allocator, list: *ArrayList(u8)) UnlimitedAllocError!void { + var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.items); + a.writer.end = list.items.len; + list.* = .empty; + defer { + list.* = .{ + .items = a.writer.buffer[0..a.writer.end], + .capacity = a.writer.buffer.len, }; - list.items.len += n; } + _ = streamRemaining(r, &a.writer) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + error.ReadFailed => return error.ReadFailed, + }; } /// Writes bytes from the internally tracked stream position to `data`. @@ -1295,7 +1255,10 @@ fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Resu /// Ensures `capacity` more data can be buffered without rebasing. pub fn rebase(r: *Reader, capacity: usize) RebaseError!void { - if (r.end + capacity <= r.buffer.len) return; + if (r.end + capacity <= r.buffer.len) { + @branchHint(.likely); + return; + } return r.vtable.rebase(r, capacity); } diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index ecf8cdba62..6c5798f7f8 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -329,7 +329,7 @@ pub fn rebase(w: *Writer, preserve: usize, unused_capacity_len: usize) Error!voi @branchHint(.likely); return; } - try w.vtable.rebase(w, preserve, unused_capacity_len); + return w.vtable.rebase(w, preserve, unused_capacity_len); } pub fn defaultRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void { @@ -2349,6 +2349,13 @@ pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Erro unreachable; } +pub fn unreachableRebase(w: *Writer, preserve: usize, capacity: usize) Error!void { + _ = w; + _ = preserve; + _ = capacity; + unreachable; +} + /// Provides a `Writer` implementation based on calling `Hasher.update`, sending /// all data also to an underlying `Writer`. /// @@ -2489,10 +2496,6 @@ pub fn Hashing(comptime Hasher: type) type { pub const Allocating = struct { allocator: Allocator, writer: Writer, - /// Every call to `drain` ensures at least this amount of unused capacity - /// before it returns. This prevents an infinite loop in interface logic - /// that calls `drain`. - minimum_unused_capacity: usize = 1, pub fn init(allocator: Allocator) Allocating { return .{ @@ -2604,13 +2607,12 @@ pub const Allocating = struct { const gpa = a.allocator; const pattern = data[data.len - 1]; const splat_len = pattern.len * splat; - const bump = a.minimum_unused_capacity; var list = a.toArrayList(); defer setArrayList(a, list); const start_len = list.items.len; assert(data.len != 0); for (data) |bytes| { - list.ensureUnusedCapacity(gpa, bytes.len + splat_len + bump) catch return error.WriteFailed; + list.ensureUnusedCapacity(gpa, bytes.len + splat_len + 1) catch return error.WriteFailed; list.appendSliceAssumeCapacity(bytes); } if (splat == 0) { @@ -2641,11 +2643,12 @@ pub const Allocating = struct { } fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void { - _ = preserve; // This implementation always preserves the entire buffer. const a: *Allocating = @fieldParentPtr("writer", w); const gpa = a.allocator; var list = a.toArrayList(); defer setArrayList(a, list); + const total = std.math.add(usize, preserve, minimum_len) catch return error.WriteFailed; + list.ensureTotalCapacity(gpa, total) catch return error.WriteFailed; list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed; } diff --git a/lib/std/array_list.zig b/lib/std/array_list.zig index 2272490996..f15e59a424 100644 --- a/lib/std/array_list.zig +++ b/lib/std/array_list.zig @@ -1033,7 +1033,7 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type { pub fn print(self: *Self, gpa: Allocator, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void { comptime assert(T == u8); try self.ensureUnusedCapacity(gpa, fmt.len); - var aw: std.io.Writer.Allocating = .fromArrayList(gpa, self); + var aw: std.Io.Writer.Allocating = .fromArrayList(gpa, self); defer self.* = aw.toArrayList(); return aw.writer.print(fmt, args) catch |err| switch (err) { error.WriteFailed => return error.OutOfMemory, diff --git a/lib/std/compress/flate/Decompress.zig b/lib/std/compress/flate/Decompress.zig index a970360a85..47570beab2 100644 --- a/lib/std/compress/flate/Decompress.zig +++ b/lib/std/compress/flate/Decompress.zig @@ -62,7 +62,7 @@ pub const Error = Container.Error || error{ const direct_vtable: Reader.VTable = .{ .stream = streamDirect, .rebase = rebaseFallible, - .discard = discard, + .discard = discardDirect, .readVec = readVec, }; @@ -105,17 +105,16 @@ fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void { fn rebase(r: *Reader, capacity: usize) void { assert(capacity <= r.buffer.len - flate.history_len); assert(r.end + capacity > r.buffer.len); - const discard_n = r.end - flate.history_len; + const discard_n = @min(r.seek, r.end - flate.history_len); const keep = r.buffer[discard_n..r.end]; @memmove(r.buffer[0..keep.len], keep); - assert(keep.len != 0); r.end = keep.len; r.seek -= discard_n; } /// This could be improved so that when an amount is discarded that includes an /// entire frame, skip decoding that frame. -fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { +fn discardDirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len); var writer: Writer = .{ .vtable = &.{ @@ -167,11 +166,14 @@ fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { fn streamIndirectInner(d: *Decompress) Reader.Error!usize { const r = &d.reader; - if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len); + if (r.buffer.len - r.end < flate.history_len) rebase(r, flate.history_len); var writer: Writer = .{ .buffer = r.buffer, .end = r.end, - .vtable = &.{ .drain = Writer.unreachableDrain }, + .vtable = &.{ + .drain = Writer.unreachableDrain, + .rebase = Writer.unreachableRebase, + }, }; defer r.end = writer.end; _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { @@ -1251,8 +1253,6 @@ test "zlib should not overshoot" { fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void { var reader: Reader = .fixed(in); var aw: Writer.Allocating = .init(testing.allocator); - aw.minimum_unused_capacity = flate.history_len; - try aw.ensureUnusedCapacity(flate.max_window_len); defer aw.deinit(); var decompress: Decompress = .init(&reader, container, &.{}); @@ -1263,8 +1263,6 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void { var in: std.Io.Reader = .fixed(compressed); var aw: std.Io.Writer.Allocating = .init(testing.allocator); - aw.minimum_unused_capacity = flate.history_len; - try aw.ensureUnusedCapacity(flate.max_window_len); defer aw.deinit(); var decompress: Decompress = .init(&in, container, &.{}); diff --git a/lib/std/compress/zstd.zig b/lib/std/compress/zstd.zig index a59df317bb..39073b51c5 100644 --- a/lib/std/compress/zstd.zig +++ b/lib/std/compress/zstd.zig @@ -78,15 +78,14 @@ pub const table_size_max = struct { }; fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 { - var out: std.ArrayListUnmanaged(u8) = .empty; - defer out.deinit(gpa); - try out.ensureUnusedCapacity(gpa, default_window_len); + var out: std.Io.Writer.Allocating = .init(gpa); + defer out.deinit(); - var in: std.io.Reader = .fixed(compressed); + var in: std.Io.Reader = .fixed(compressed); var zstd_stream: Decompress = .init(&in, &.{}, .{}); - try zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited); + _ = try zstd_stream.reader.streamRemaining(&out.writer); - return out.toOwnedSlice(gpa); + return out.toOwnedSlice(); } fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void { @@ -99,15 +98,14 @@ fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void { const gpa = std.testing.allocator; - var out: std.ArrayListUnmanaged(u8) = .empty; - defer out.deinit(gpa); - try out.ensureUnusedCapacity(gpa, default_window_len); + var out: std.Io.Writer.Allocating = .init(gpa); + defer out.deinit(); - var in: std.io.Reader = .fixed(compressed); + var in: std.Io.Reader = .fixed(compressed); var zstd_stream: Decompress = .init(&in, &.{}, .{}); try std.testing.expectError( error.ReadFailed, - zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited), + zstd_stream.reader.streamRemaining(&out.writer), ); try std.testing.expectError(err, zstd_stream.err orelse {}); } diff --git a/lib/std/compress/zstd/Decompress.zig b/lib/std/compress/zstd/Decompress.zig index aacfc70171..08fda244f9 100644 --- a/lib/std/compress/zstd/Decompress.zig +++ b/lib/std/compress/zstd/Decompress.zig @@ -73,6 +73,20 @@ pub const Error = error{ WindowSizeUnknown, }; +const direct_vtable: Reader.VTable = .{ + .stream = streamDirect, + .rebase = rebaseFallible, + .discard = discardDirect, + .readVec = readVec, +}; + +const indirect_vtable: Reader.VTable = .{ + .stream = streamIndirect, + .rebase = rebaseFallible, + .discard = discardIndirect, + .readVec = readVec, +}; + /// When connecting `reader` to a `Writer`, `buffer` should be empty, and /// `Writer.buffer` capacity has requirements based on `Options.window_len`. /// @@ -84,12 +98,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress { .verify_checksum = options.verify_checksum, .window_len = options.window_len, .reader = .{ - .vtable = &.{ - .stream = stream, - .rebase = rebase, - .discard = discard, - .readVec = readVec, - }, + .vtable = if (buffer.len == 0) &direct_vtable else &indirect_vtable, .buffer = buffer, .seek = 0, .end = 0, @@ -97,11 +106,27 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress { }; } -fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void { +fn streamDirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + return stream(d, w, limit); +} + +fn streamIndirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize { + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + _ = limit; + _ = w; + return streamIndirectInner(d); +} + +fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void { + rebase(r, capacity); +} + +fn rebase(r: *Reader, capacity: usize) void { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); assert(capacity <= r.buffer.len - d.window_len); assert(r.end + capacity > r.buffer.len); - const discard_n = r.end - d.window_len; + const discard_n = @min(r.seek, r.end - d.window_len); const keep = r.buffer[discard_n..r.end]; @memmove(r.buffer[0..keep.len], keep); r.end = keep.len; @@ -110,9 +135,9 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void { /// This could be improved so that when an amount is discarded that includes an /// entire frame, skip decoding that frame. -fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { +fn discardDirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); - r.rebase(d.window_len) catch unreachable; + rebase(r, d.window_len); var writer: Writer = .{ .vtable = &.{ .drain = std.Io.Writer.Discarding.drain, @@ -134,25 +159,53 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { return n; } -fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { - _ = data; +fn discardIndirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize { const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); - assert(r.seek == r.end); - r.rebase(d.window_len) catch unreachable; + rebase(r, d.window_len); var writer: Writer = .{ .buffer = r.buffer, .end = r.end, - .vtable = &.{ .drain = Writer.fixedDrain }, + .vtable = &.{ .drain = Writer.unreachableDrain }, }; - r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { + { + defer r.end = writer.end; + _ = stream(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { + error.WriteFailed => unreachable, + else => |e| return e, + }; + } + const n = limit.minInt(r.end - r.seek); + r.seek += n; + return n; +} + +fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize { + _ = data; + const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); + return streamIndirectInner(d); +} + +fn streamIndirectInner(d: *Decompress) Reader.Error!usize { + const r = &d.reader; + if (r.buffer.len - r.end < zstd.block_size_max) rebase(r, zstd.block_size_max); + assert(r.buffer.len - r.end >= zstd.block_size_max); + var writer: Writer = .{ + .buffer = r.buffer, + .end = r.end, + .vtable = &.{ + .drain = Writer.unreachableDrain, + .rebase = Writer.unreachableRebase, + }, + }; + defer r.end = writer.end; + _ = stream(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) { error.WriteFailed => unreachable, else => |e| return e, }; return 0; } -fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize { - const d: *Decompress = @alignCast(@fieldParentPtr("reader", r)); +fn stream(d: *Decompress, w: *Writer, limit: Limit) Reader.StreamError!usize { const in = d.input; state: switch (d.state) { @@ -170,7 +223,7 @@ fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize { else => |e| return e, }; const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little); - initFrame(d, w.buffer.len, magic) catch |err| { + initFrame(d, magic) catch |err| { d.err = err; return error.ReadFailed; }; @@ -198,13 +251,13 @@ fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize { } } -fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void { +fn initFrame(d: *Decompress, magic: Frame.Magic) !void { const in = d.input; switch (magic.kind() orelse return error.BadMagic) { .zstandard => { const header = try Frame.Zstandard.Header.decode(in); d.state = .{ .in_frame = .{ - .frame = try Frame.init(header, window_size_max, d.verify_checksum), + .frame = try Frame.init(header, d.window_len, d.verify_checksum), .checksum = null, .decompressed_size = 0, .decode = .init, @@ -258,7 +311,6 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame) try decode.readInitialFseState(&bit_stream); // Ensures the following calls to `decodeSequence` will not flush. - if (window_len + frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize; const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max]; const write_pos = dest.ptr - w.buffer.ptr; for (0..sequences_header.sequence_count - 1) |_| { @@ -775,7 +827,6 @@ pub const Frame = struct { try w.splatByteAll(d.literal_streams.one[0], len); }, .compressed, .treeless => { - if (len > w.buffer.len) return error.OutputBufferUndersize; const buf = try w.writableSlice(len); const huffman_tree = d.huffman_tree.?; const max_bit_count = huffman_tree.max_bit_count; diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index 2cdeede7b1..01c672b3a0 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2247,7 +2247,7 @@ pub const ElfModule = struct { var decompress: std.compress.flate.Decompress = .init(§ion_reader, .zlib, &.{}); var decompressed_section: ArrayList(u8) = .empty; defer decompressed_section.deinit(gpa); - decompress.reader.appendRemainingUnlimited(gpa, null, &decompressed_section, std.compress.flate.history_len) catch { + decompress.reader.appendRemainingUnlimited(gpa, &decompressed_section) catch { invalidDebugInfoDetected(); continue; }; diff --git a/lib/std/http/test.zig b/lib/std/http/test.zig index 20c7383650..91a57dbbe2 100644 --- a/lib/std/http/test.zig +++ b/lib/std/http/test.zig @@ -149,9 +149,8 @@ test "HTTP server handles a chunked transfer coding request" { "content-type: text/plain\r\n" ++ "\r\n" ++ "message from server!\n"; - var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded - var stream_reader = stream.reader(&tiny_buffer); - const response = try stream_reader.interface().allocRemaining(gpa, .limited(expected_response.len)); + var stream_reader = stream.reader(&.{}); + const response = try stream_reader.interface().allocRemaining(gpa, .limited(expected_response.len + 1)); defer gpa.free(response); try expectEqualStrings(expected_response, response); } @@ -293,8 +292,7 @@ test "Server.Request.respondStreaming non-chunked, unknown content-length" { var stream_writer = stream.writer(&.{}); try stream_writer.interface.writeAll(request_bytes); - var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded - var stream_reader = stream.reader(&tiny_buffer); + var stream_reader = stream.reader(&.{}); const response = try stream_reader.interface().allocRemaining(gpa, .unlimited); defer gpa.free(response); @@ -364,8 +362,7 @@ test "receiving arbitrary http headers from the client" { var stream_writer = stream.writer(&.{}); try stream_writer.interface.writeAll(request_bytes); - var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded - var stream_reader = stream.reader(&tiny_buffer); + var stream_reader = stream.reader(&.{}); const response = try stream_reader.interface().allocRemaining(gpa, .unlimited); defer gpa.free(response); diff --git a/lib/std/unicode.zig b/lib/std/unicode.zig index 0f849fb8b1..fb0ce509df 100644 --- a/lib/std/unicode.zig +++ b/lib/std/unicode.zig @@ -4,6 +4,7 @@ const assert = std.debug.assert; const testing = std.testing; const mem = std.mem; const native_endian = builtin.cpu.arch.endian(); +const Allocator = std.mem.Allocator; /// Use this to replace an unknown, unrecognized, or unrepresentable character. /// @@ -921,7 +922,7 @@ fn utf16LeToUtf8ArrayListImpl( comptime surrogates: Surrogates, ) (switch (surrogates) { .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError, - .can_encode_surrogate_half => mem.Allocator.Error, + .can_encode_surrogate_half => Allocator.Error, })!void { assert(result.unusedCapacitySlice().len >= utf16le.len); @@ -965,15 +966,15 @@ fn utf16LeToUtf8ArrayListImpl( } } -pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error; +pub const Utf16LeToUtf8AllocError = Allocator.Error || Utf16LeToUtf8Error; pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void { try result.ensureUnusedCapacity(utf16le.len); return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half); } -/// Caller must free returned memory. -pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 { +/// Caller owns returned memory. +pub fn utf16LeToUtf8Alloc(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 { // optimistically guess that it will all be ascii. var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len); errdefer result.deinit(); @@ -982,8 +983,8 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L return result.toOwnedSlice(); } -/// Caller must free returned memory. -pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 { +/// Caller owns returned memory. +pub fn utf16LeToUtf8AllocZ(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 { // optimistically guess that it will all be ascii (and allocate space for the null terminator) var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len + 1); errdefer result.deinit(); @@ -1160,7 +1161,7 @@ pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []cons return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half); } -pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 { +pub fn utf8ToUtf16LeAlloc(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 { // optimistically guess that it will not require surrogate pairs var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len); errdefer result.deinit(); @@ -1169,7 +1170,7 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv return result.toOwnedSlice(); } -pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 { +pub fn utf8ToUtf16LeAllocZ(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 { // optimistically guess that it will not require surrogate pairs var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len + 1); errdefer result.deinit(); @@ -1750,13 +1751,13 @@ pub const Wtf8Iterator = struct { } }; -pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) mem.Allocator.Error!void { +pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Allocator.Error!void { try result.ensureUnusedCapacity(utf16le.len); return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half); } /// Caller must free returned memory. -pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Allocator.Error![]u8 { +pub fn wtf16LeToWtf8Alloc(allocator: Allocator, wtf16le: []const u16) Allocator.Error![]u8 { // optimistically guess that it will all be ascii. var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len); errdefer result.deinit(); @@ -1766,7 +1767,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al } /// Caller must free returned memory. -pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.Allocator.Error![:0]u8 { +pub fn wtf16LeToWtf8AllocZ(allocator: Allocator, wtf16le: []const u16) Allocator.Error![:0]u8 { // optimistically guess that it will all be ascii (and allocate space for the null terminator) var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len + 1); errdefer result.deinit(); @@ -1784,7 +1785,7 @@ pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []cons return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half); } -pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 { +pub fn wtf8ToWtf16LeAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 { // optimistically guess that it will not require surrogate pairs var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len); errdefer result.deinit(); @@ -1793,7 +1794,7 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv return result.toOwnedSlice(); } -pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 { +pub fn wtf8ToWtf16LeAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 { // optimistically guess that it will not require surrogate pairs var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len + 1); errdefer result.deinit(); @@ -1870,7 +1871,7 @@ pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void { } } -pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 { +pub fn wtf8ToUtf8LossyAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 { const utf8 = try allocator.alloc(u8, wtf8.len); errdefer allocator.free(utf8); @@ -1879,7 +1880,7 @@ pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ I return utf8; } -pub fn wtf8ToUtf8LossyAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 { +pub fn wtf8ToUtf8LossyAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 { const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0); errdefer allocator.free(utf8); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index cb4be0de64..cbb01b2b38 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -554,8 +554,11 @@ test isUnderscore { try std.testing.expect(!isUnderscore("\\x5f")); } +/// If the source can be UTF-16LE encoded, this function asserts that `gpa` +/// will align a byte-sized allocation to at least 2. Allocators that don't do +/// this are rare. pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader) ![:0]u8 { - var buffer: std.ArrayListAlignedUnmanaged(u8, .@"2") = .empty; + var buffer: std.ArrayList(u8) = .empty; defer buffer.deinit(gpa); if (file_reader.getSize()) |size| { @@ -564,7 +567,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader try buffer.ensureTotalCapacityPrecise(gpa, casted_size + 1); } else |_| {} - try file_reader.interface.appendRemaining(gpa, .@"2", &buffer, .limited(max_src_size)); + try file_reader.interface.appendRemaining(gpa, &buffer, .limited(max_src_size)); // Detect unsupported file types with their Byte Order Mark const unsupported_boms = [_][]const u8{ @@ -581,7 +584,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8 if (std.mem.startsWith(u8, buffer.items, "\xff\xfe")) { if (buffer.items.len % 2 != 0) return error.InvalidEncoding; - return std.unicode.utf16LeToUtf8AllocZ(gpa, @ptrCast(buffer.items)) catch |err| switch (err) { + return std.unicode.utf16LeToUtf8AllocZ(gpa, @ptrCast(@alignCast(buffer.items))) catch |err| switch (err) { error.DanglingSurrogateHalf => error.UnsupportedEncoding, error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding, error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,