commit 483c057a771aca3b045ec8148fa5fb1a5e28fa17 (tree)
parent 3468872d8376602fdbfdb5b0d6d582c278ad1990
Author: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 11 Jan 2021 16:52:29 -0700
Merge branch 'clean up writeFileAllUnseekable by using readers'
closes #7156
Diffstat:
5 files changed, 66 insertions(+), 20 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -375,10 +375,11 @@ set(ZIG_STAGE2_SOURCES
"${CMAKE_SOURCE_DIR}/lib/std/io/buffered_atomic_file.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/buffered_writer.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/change_detection_stream.zig"
- "${CMAKE_SOURCE_DIR}/lib/std/io/counting_writer.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/counting_reader.zig"
+ "${CMAKE_SOURCE_DIR}/lib/std/io/counting_writer.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/find_byte_writer.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/fixed_buffer_stream.zig"
+ "${CMAKE_SOURCE_DIR}/lib/std/io/limited_reader.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/reader.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/seekable_stream.zig"
"${CMAKE_SOURCE_DIR}/lib/std/io/writer.zig"
diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig
@@ -698,7 +698,7 @@ pub const File = struct {
header_count: usize = 0,
};
- pub const WriteFileError = ReadError || WriteError;
+ pub const WriteFileError = ReadError || error{EndOfStream} || WriteError;
pub fn writeFileAll(self: File, in_file: File, args: WriteFileOptions) WriteFileError!void {
return self.writeFileAllSendfile(in_file, args) catch |err| switch (err) {
@@ -722,23 +722,14 @@ pub const File = struct {
try self.writevAll(headers);
- var buffer: [4096]u8 = undefined;
- {
- var index: usize = 0;
- // Skip in_offset bytes.
- while (index < args.in_offset) {
- const ask = math.min(buffer.len, args.in_offset - index);
- const amt = try in_file.read(buffer[0..ask]);
- index += amt;
- }
- }
- const in_len = args.in_len orelse math.maxInt(u64);
- var index: usize = 0;
- while (index < in_len) {
- const ask = math.min(buffer.len, in_len - index);
- const amt = try in_file.read(buffer[0..ask]);
- if (amt == 0) break;
- index += try self.write(buffer[0..amt]);
+ try in_file.reader().skipBytes(args.in_offset, .{ .buf_size = 4096 });
+
+ var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
+ if (args.in_len) |len| {
+ var stream = std.io.limitedReader(in_file.reader(), len);
+ try fifo.pump(stream.reader(), self.writer());
+ } else {
+ try fifo.pump(in_file.reader(), self.writer());
}
try self.writevAll(trailers);
diff --git a/lib/std/io.zig b/lib/std/io.zig
@@ -125,6 +125,9 @@ pub const fixedBufferStream = @import("io/fixed_buffer_stream.zig").fixedBufferS
pub const CWriter = @import("io/c_writer.zig").CWriter;
pub const cWriter = @import("io/c_writer.zig").cWriter;
+pub const LimitedReader = @import("io/limited_reader.zig").LimitedReader;
+pub const limitedReader = @import("io/limited_reader.zig").limitedReader;
+
pub const CountingWriter = @import("io/counting_writer.zig").CountingWriter;
pub const countingWriter = @import("io/counting_writer.zig").countingWriter;
pub const CountingReader = @import("io/counting_reader.zig").CountingReader;
diff --git a/lib/std/io/limited_reader.zig b/lib/std/io/limited_reader.zig
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2015-2020 Zig Contributors
+// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
+// The MIT license requires this copyright notice to be included in all copies
+// and substantial portions of the software.
+const std = @import("../std.zig");
+const io = std.io;
+const assert = std.debug.assert;
+const testing = std.testing;
+
+pub fn LimitedReader(comptime ReaderType: type) type {
+ return struct {
+ inner_reader: ReaderType,
+ bytes_left: u64,
+
+ pub const Error = ReaderType.Error;
+ pub const Reader = io.Reader(*Self, Error, read);
+
+ const Self = @This();
+
+ pub fn read(self: *Self, dest: []u8) Error!usize {
+ const max_read = std.math.min(self.bytes_left, dest.len);
+ const n = try self.inner_reader.read(dest[0..max_read]);
+ self.bytes_left -= n;
+ return n;
+ }
+
+ pub fn reader(self: *Self) Reader {
+ return .{ .context = self };
+ }
+ };
+}
+
+/// Returns an initialised `LimitedReader`
+/// `bytes_left` is a `u64` to be able to take 64 bit file offsets
+pub fn limitedReader(inner_reader: anytype, bytes_left: u64) LimitedReader(@TypeOf(inner_reader)) {
+ return .{ .inner_reader = inner_reader, .bytes_left = bytes_left };
+}
+
+test "basic usage" {
+ const data = "hello world";
+ var fbs = std.io.fixedBufferStream(data);
+ var early_stream = limitedReader(fbs.reader(), 3);
+
+ var buf: [5]u8 = undefined;
+ testing.expectEqual(@as(usize, 3), try early_stream.reader().read(&buf));
+ testing.expectEqualSlices(u8, data[0..3], buf[0..3]);
+ testing.expectEqual(@as(usize, 0), try early_stream.reader().read(&buf));
+ testing.expectError(error.EndOfStream, early_stream.reader().skipBytes(10, .{}));
+}
diff --git a/lib/std/io/reader.zig b/lib/std/io/reader.zig
@@ -271,8 +271,9 @@ pub fn Reader(
buf_size: usize = 512,
};
+ // `num_bytes` is a `u64` to match `off_t`
/// Reads `num_bytes` bytes from the stream and discards them
- pub fn skipBytes(self: Self, num_bytes: usize, comptime options: SkipBytesOptions) !void {
+ pub fn skipBytes(self: Self, num_bytes: u64, comptime options: SkipBytesOptions) !void {
var buf: [options.buf_size]u8 = undefined;
var remaining = num_bytes;
