commit f34b4780b7bd52d14df253d0762d9c73db8eb226 (tree)
parent c41ac8f19ec96ed854d9ac31a2015006ba3e4657
Author: Andrew Kelley <andrew@ziglang.org>
Date: Tue, 22 Jul 2025 18:40:11 +0200
Merge pull request #24521 from ziglang/fs-streaming
std.fs.File: delete writeFileAll and friends
Diffstat:
19 files changed, 722 insertions(+), 1945 deletions(-)
diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig
@@ -708,7 +708,7 @@ fn runStepNames(
const total_count = success_count + failure_count + pending_count + skipped_count;
ttyconf.setColor(w, .cyan) catch {};
- w.writeAll("Build Summary:") catch {};
+ w.writeAll("\nBuild Summary:") catch {};
ttyconf.setColor(w, .reset) catch {};
w.print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {};
if (skipped_count > 0) w.print("; {d} skipped", .{skipped_count}) catch {};
diff --git a/lib/compiler/objcopy.zig b/lib/compiler/objcopy.zig
@@ -13,6 +13,9 @@ const Server = std.zig.Server;
var stdin_buffer: [1024]u8 = undefined;
var stdout_buffer: [1024]u8 = undefined;
+var input_buffer: [1024]u8 = undefined;
+var output_buffer: [1024]u8 = undefined;
+
pub fn main() !void {
var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);
defer arena_instance.deinit();
@@ -145,13 +148,16 @@ fn cmdObjCopy(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
const input = opt_input orelse fatal("expected input parameter", .{});
const output = opt_output orelse fatal("expected output parameter", .{});
- var in_file = fs.cwd().openFile(input, .{}) catch |err|
- fatal("unable to open '{s}': {s}", .{ input, @errorName(err) });
- defer in_file.close();
+ const input_file = fs.cwd().openFile(input, .{}) catch |err| fatal("failed to open {s}: {t}", .{ input, err });
+ defer input_file.close();
+
+ const stat = input_file.stat() catch |err| fatal("failed to stat {s}: {t}", .{ input, err });
- const elf_hdr = std.elf.Header.read(in_file) catch |err| switch (err) {
- error.InvalidElfMagic => fatal("not an ELF file: '{s}'", .{input}),
- else => fatal("unable to read '{s}': {s}", .{ input, @errorName(err) }),
+ var in: File.Reader = .initSize(input_file, &input_buffer, stat.size);
+
+ const elf_hdr = std.elf.Header.read(&in.interface) catch |err| switch (err) {
+ error.ReadFailed => fatal("unable to read {s}: {t}", .{ input, in.err.? }),
+ else => |e| fatal("invalid elf file: {t}", .{e}),
};
const in_ofmt = .elf;
@@ -168,16 +174,12 @@ fn cmdObjCopy(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
}
};
- const mode = mode: {
- if (out_fmt != .elf or only_keep_debug)
- break :mode fs.File.default_mode;
- if (in_file.stat()) |stat|
- break :mode stat.mode
- else |_|
- break :mode fs.File.default_mode;
- };
- var out_file = try fs.cwd().createFile(output, .{ .mode = mode });
- defer out_file.close();
+ const mode = if (out_fmt != .elf or only_keep_debug) fs.File.default_mode else stat.mode;
+
+ var output_file = try fs.cwd().createFile(output, .{ .mode = mode });
+ defer output_file.close();
+
+ var out = output_file.writer(&output_buffer);
switch (out_fmt) {
.hex, .raw => {
@@ -192,7 +194,7 @@ fn cmdObjCopy(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
if (set_section_flags != null)
fatal("zig objcopy: ELF to RAW or HEX copying does not support --set_section_flags", .{});
- try emitElf(arena, in_file, out_file, elf_hdr, .{
+ try emitElf(arena, &in, &out, elf_hdr, .{
.ofmt = out_fmt,
.only_section = only_section,
.pad_to = pad_to,
@@ -208,22 +210,13 @@ fn cmdObjCopy(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
if (pad_to) |_|
fatal("zig objcopy: ELF to ELF copying does not support --pad-to", .{});
- try stripElf(arena, in_file, out_file, elf_hdr, .{
- .strip_debug = strip_debug,
- .strip_all = strip_all,
- .only_keep_debug = only_keep_debug,
- .add_debuglink = opt_add_debuglink,
- .extract_to = opt_extract,
- .compress_debug = compress_debug_sections,
- .add_section = add_section,
- .set_section_alignment = set_section_alignment,
- .set_section_flags = set_section_flags,
- });
- return std.process.cleanExit();
+ fatal("unimplemented", .{});
},
else => fatal("unsupported output object format: {s}", .{@tagName(out_fmt)}),
}
+ try out.end();
+
if (listen) {
var stdin_reader = fs.File.stdin().reader(&stdin_buffer);
var stdout_writer = fs.File.stdout().writer(&stdout_buffer);
@@ -304,12 +297,12 @@ const SetSectionFlags = struct {
fn emitElf(
arena: Allocator,
- in_file: File,
- out_file: File,
+ in: *File.Reader,
+ out: *File.Writer,
elf_hdr: elf.Header,
options: EmitRawElfOptions,
) !void {
- var binary_elf_output = try BinaryElfOutput.parse(arena, in_file, elf_hdr);
+ var binary_elf_output = try BinaryElfOutput.parse(arena, in, elf_hdr);
defer binary_elf_output.deinit();
if (options.ofmt == .elf) {
@@ -328,8 +321,8 @@ fn emitElf(
continue;
}
- try writeBinaryElfSection(in_file, out_file, section);
- try padFile(out_file, options.pad_to);
+ try writeBinaryElfSection(in, out, section);
+ try padFile(out, options.pad_to);
return;
}
},
@@ -342,10 +335,10 @@ fn emitElf(
switch (options.ofmt) {
.raw => {
for (binary_elf_output.sections.items) |section| {
- try out_file.seekTo(section.binaryOffset);
- try writeBinaryElfSection(in_file, out_file, section);
+ try out.seekTo(section.binaryOffset);
+ try writeBinaryElfSection(in, out, section);
}
- try padFile(out_file, options.pad_to);
+ try padFile(out, options.pad_to);
},
.hex => {
if (binary_elf_output.segments.items.len == 0) return;
@@ -353,15 +346,15 @@ fn emitElf(
return error.InvalidHexfileAddressRange;
}
- var hex_writer = HexWriter{ .out_file = out_file };
+ var hex_writer = HexWriter{ .out = out };
for (binary_elf_output.segments.items) |segment| {
- try hex_writer.writeSegment(segment, in_file);
+ try hex_writer.writeSegment(segment, in);
}
if (options.pad_to) |_| {
// Padding to a size in hex files isn't applicable
return error.InvalidArgument;
}
- try hex_writer.writeEOF();
+ try hex_writer.writeEof();
},
else => unreachable,
}
@@ -399,7 +392,7 @@ const BinaryElfOutput = struct {
self.segments.deinit(self.allocator);
}
- pub fn parse(allocator: Allocator, elf_file: File, elf_hdr: elf.Header) !Self {
+ pub fn parse(allocator: Allocator, in: *File.Reader, elf_hdr: elf.Header) !Self {
var self: Self = .{
.segments = .{},
.sections = .{},
@@ -412,7 +405,7 @@ const BinaryElfOutput = struct {
self.shstrtab = blk: {
if (elf_hdr.shstrndx >= elf_hdr.shnum) break :blk null;
- var section_headers = elf_hdr.section_header_iterator(&elf_file);
+ var section_headers = elf_hdr.iterateSectionHeaders(in);
var section_counter: usize = 0;
while (section_counter < elf_hdr.shstrndx) : (section_counter += 1) {
@@ -421,18 +414,13 @@ const BinaryElfOutput = struct {
const shstrtab_shdr = (try section_headers.next()).?;
- const buffer = try allocator.alloc(u8, @intCast(shstrtab_shdr.sh_size));
- errdefer allocator.free(buffer);
-
- const num_read = try elf_file.preadAll(buffer, shstrtab_shdr.sh_offset);
- if (num_read != buffer.len) return error.EndOfStream;
-
- break :blk buffer;
+ try in.seekTo(shstrtab_shdr.sh_offset);
+ break :blk try in.interface.readAlloc(allocator, shstrtab_shdr.sh_size);
};
errdefer if (self.shstrtab) |shstrtab| allocator.free(shstrtab);
- var section_headers = elf_hdr.section_header_iterator(&elf_file);
+ var section_headers = elf_hdr.iterateSectionHeaders(in);
while (try section_headers.next()) |section| {
if (sectionValidForOutput(section)) {
const newSection = try allocator.create(BinaryElfSection);
@@ -451,7 +439,7 @@ const BinaryElfOutput = struct {
}
}
- var program_headers = elf_hdr.program_header_iterator(&elf_file);
+ var program_headers = elf_hdr.iterateProgramHeaders(in);
while (try program_headers.next()) |phdr| {
if (phdr.p_type == elf.PT_LOAD) {
const newSegment = try allocator.create(BinaryElfSegment);
@@ -539,19 +527,17 @@ const BinaryElfOutput = struct {
}
};
-fn writeBinaryElfSection(elf_file: File, out_file: File, section: *BinaryElfSection) !void {
- try out_file.writeFileAll(elf_file, .{
- .in_offset = section.elfOffset,
- .in_len = section.fileSize,
- });
+fn writeBinaryElfSection(in: *File.Reader, out: *File.Writer, section: *BinaryElfSection) !void {
+ try in.seekTo(section.elfOffset);
+ _ = try out.interface.sendFileAll(in, .limited(section.fileSize));
}
const HexWriter = struct {
prev_addr: ?u32 = null,
- out_file: File,
+ out: *File.Writer,
/// Max data bytes per line of output
- const MAX_PAYLOAD_LEN: u8 = 16;
+ const max_payload_len: u8 = 16;
fn addressParts(address: u16) [2]u8 {
const msb: u8 = @truncate(address >> 8);
@@ -627,13 +613,13 @@ const HexWriter = struct {
return (sum ^ 0xFF) +% 1;
}
- fn write(self: Record, file: File) File.WriteError!void {
+ fn write(self: Record, out: *File.Writer) !void {
const linesep = "\r\n";
// colon, (length, address, type, payload, checksum) as hex, CRLF
- const BUFSIZE = 1 + (1 + 2 + 1 + MAX_PAYLOAD_LEN + 1) * 2 + linesep.len;
+ const BUFSIZE = 1 + (1 + 2 + 1 + max_payload_len + 1) * 2 + linesep.len;
var outbuf: [BUFSIZE]u8 = undefined;
const payload_bytes = self.getPayloadBytes();
- assert(payload_bytes.len <= MAX_PAYLOAD_LEN);
+ assert(payload_bytes.len <= max_payload_len);
const line = try std.fmt.bufPrint(&outbuf, ":{0X:0>2}{1X:0>4}{2X:0>2}{3X}{4X:0>2}" ++ linesep, .{
@as(u8, @intCast(payload_bytes.len)),
@@ -642,38 +628,37 @@ const HexWriter = struct {
payload_bytes,
self.checksum(),
});
- try file.writeAll(line);
+ try out.interface.writeAll(line);
}
};
- pub fn writeSegment(self: *HexWriter, segment: *const BinaryElfSegment, elf_file: File) !void {
- var buf: [MAX_PAYLOAD_LEN]u8 = undefined;
+ pub fn writeSegment(self: *HexWriter, segment: *const BinaryElfSegment, in: *File.Reader) !void {
+ var buf: [max_payload_len]u8 = undefined;
var bytes_read: usize = 0;
while (bytes_read < segment.fileSize) {
const row_address: u32 = @intCast(segment.physicalAddress + bytes_read);
const remaining = segment.fileSize - bytes_read;
- const to_read: usize = @intCast(@min(remaining, MAX_PAYLOAD_LEN));
- const did_read = try elf_file.preadAll(buf[0..to_read], segment.elfOffset + bytes_read);
- if (did_read < to_read) return error.UnexpectedEOF;
+ const dest = buf[0..@min(remaining, max_payload_len)];
+ try in.seekTo(segment.elfOffset + bytes_read);
+ try in.interface.readSliceAll(dest);
+ try self.writeDataRow(row_address, dest);
- try self.writeDataRow(row_address, buf[0..did_read]);
-
- bytes_read += did_read;
+ bytes_read += dest.len;
}
}
- fn writeDataRow(self: *HexWriter, address: u32, data: []const u8) File.WriteError!void {
+ fn writeDataRow(self: *HexWriter, address: u32, data: []const u8) !void {
const record = Record.Data(address, data);
if (address > 0xFFFF and (self.prev_addr == null or record.address != self.prev_addr.?)) {
- try Record.Address(address).write(self.out_file);
+ try Record.Address(address).write(self.out);
}
- try record.write(self.out_file);
+ try record.write(self.out);
self.prev_addr = @intCast(record.address + data.len);
}
- fn writeEOF(self: HexWriter) File.WriteError!void {
- try Record.EOF().write(self.out_file);
+ fn writeEof(self: HexWriter) !void {
+ try Record.EOF().write(self.out);
}
};
@@ -686,9 +671,9 @@ fn containsValidAddressRange(segments: []*BinaryElfSegment) bool {
return true;
}
-fn padFile(f: File, opt_size: ?u64) !void {
+fn padFile(out: *File.Writer, opt_size: ?u64) !void {
const size = opt_size orelse return;
- try f.setEndPos(size);
+ try out.file.setEndPos(size);
}
test "HexWriter.Record.Address has correct payload and checksum" {
@@ -732,836 +717,6 @@ test "containsValidAddressRange" {
try std.testing.expect(containsValidAddressRange(&buf));
}
-// -------------
-// ELF to ELF stripping
-
-const StripElfOptions = struct {
- extract_to: ?[]const u8 = null,
- add_debuglink: ?[]const u8 = null,
- strip_all: bool = false,
- strip_debug: bool = false,
- only_keep_debug: bool = false,
- compress_debug: bool = false,
- add_section: ?AddSection,
- set_section_alignment: ?SetSectionAlignment,
- set_section_flags: ?SetSectionFlags,
-};
-
-fn stripElf(
- allocator: Allocator,
- in_file: File,
- out_file: File,
- elf_hdr: elf.Header,
- options: StripElfOptions,
-) !void {
- const Filter = ElfFileHelper.Filter;
- const DebugLink = ElfFileHelper.DebugLink;
-
- const filter: Filter = filter: {
- if (options.only_keep_debug) break :filter .debug;
- if (options.strip_all) break :filter .program;
- if (options.strip_debug) break :filter .program_and_symbols;
- break :filter .all;
- };
-
- const filter_complement: ?Filter = blk: {
- if (options.extract_to) |_| {
- break :blk switch (filter) {
- .program => .debug_and_symbols,
- .debug => .program_and_symbols,
- .program_and_symbols => .debug,
- .debug_and_symbols => .program,
- .all => fatal("zig objcopy: nothing to extract", .{}),
- };
- } else {
- break :blk null;
- }
- };
- const debuglink_path = path: {
- if (options.add_debuglink) |path| break :path path;
- if (options.extract_to) |path| break :path path;
- break :path null;
- };
-
- switch (elf_hdr.is_64) {
- inline else => |is_64| {
- var elf_file = try ElfFile(is_64).parse(allocator, in_file, elf_hdr);
- defer elf_file.deinit();
-
- if (options.add_section) |user_section| {
- for (elf_file.sections) |section| {
- if (std.mem.eql(u8, section.name, user_section.section_name)) {
- fatal("zig objcopy: unable to add section '{s}'. Section already exists in input", .{user_section.section_name});
- }
- }
- }
-
- if (filter_complement) |flt| {
- // write the .dbg file and close it, so it can be read back to compute the debuglink checksum.
- const path = options.extract_to.?;
- const dbg_file = std.fs.cwd().createFile(path, .{}) catch |err| {
- fatal("zig objcopy: unable to create '{s}': {s}", .{ path, @errorName(err) });
- };
- defer dbg_file.close();
-
- try elf_file.emit(allocator, dbg_file, in_file, .{ .section_filter = flt, .compress_debug = options.compress_debug });
- }
-
- const debuglink: ?DebugLink = if (debuglink_path) |path| ElfFileHelper.createDebugLink(path) else null;
- try elf_file.emit(allocator, out_file, in_file, .{
- .section_filter = filter,
- .debuglink = debuglink,
- .compress_debug = options.compress_debug,
- .add_section = options.add_section,
- .set_section_alignment = options.set_section_alignment,
- .set_section_flags = options.set_section_flags,
- });
- },
- }
-}
-
-// note: this is "a minimal effort implementation"
-// It doesn't support all possibile elf files: some sections type may need fixups, the program header may need fix up, ...
-// It was written for a specific use case (strip debug info to a sperate file, for linux 64-bits executables built with `zig` or `zig c++` )
-// It moves and reoders the sections as little as possible to avoid having to do fixups.
-// TODO: support non-native endianess
-
-fn ElfFile(comptime is_64: bool) type {
- const Elf_Ehdr = if (is_64) elf.Elf64_Ehdr else elf.Elf32_Ehdr;
- const Elf_Phdr = if (is_64) elf.Elf64_Phdr else elf.Elf32_Phdr;
- const Elf_Shdr = if (is_64) elf.Elf64_Shdr else elf.Elf32_Shdr;
- const Elf_Chdr = if (is_64) elf.Elf64_Chdr else elf.Elf32_Chdr;
- const Elf_Sym = if (is_64) elf.Elf64_Sym else elf.Elf32_Sym;
- const Elf_OffSize = if (is_64) elf.Elf64_Off else elf.Elf32_Off;
-
- return struct {
- raw_elf_header: Elf_Ehdr,
- program_segments: []const Elf_Phdr,
- sections: []const Section,
- arena: std.heap.ArenaAllocator,
-
- const SectionCategory = ElfFileHelper.SectionCategory;
- const section_memory_align: std.mem.Alignment = .of(Elf_Sym); // most restrictive of what we may load in memory
- const Section = struct {
- section: Elf_Shdr,
- name: []const u8 = "",
- segment: ?*const Elf_Phdr = null, // if the section is used by a program segment (there can be more than one)
- payload: ?[]align(section_memory_align.toByteUnits()) const u8 = null, // if we need the data in memory
- category: SectionCategory = .none, // should the section be kept in the exe or stripped to the debug database, or both.
- };
-
- const Self = @This();
-
- pub fn parse(gpa: Allocator, in_file: File, header: elf.Header) !Self {
- var arena = std.heap.ArenaAllocator.init(gpa);
- errdefer arena.deinit();
- const allocator = arena.allocator();
-
- var raw_header: Elf_Ehdr = undefined;
- {
- const bytes_read = try in_file.preadAll(std.mem.asBytes(&raw_header), 0);
- if (bytes_read < @sizeOf(Elf_Ehdr))
- return error.TRUNCATED_ELF;
- }
-
- // program header: list of segments
- const program_segments = blk: {
- if (@sizeOf(Elf_Phdr) != header.phentsize)
- fatal("zig objcopy: unsupported ELF file, unexpected phentsize ({d})", .{header.phentsize});
-
- const program_header = try allocator.alloc(Elf_Phdr, header.phnum);
- const bytes_read = try in_file.preadAll(std.mem.sliceAsBytes(program_header), header.phoff);
- if (bytes_read < @sizeOf(Elf_Phdr) * header.phnum)
- return error.TRUNCATED_ELF;
- break :blk program_header;
- };
-
- // section header
- const sections = blk: {
- if (@sizeOf(Elf_Shdr) != header.shentsize)
- fatal("zig objcopy: unsupported ELF file, unexpected shentsize ({d})", .{header.shentsize});
-
- const section_header = try allocator.alloc(Section, header.shnum);
-
- const raw_section_header = try allocator.alloc(Elf_Shdr, header.shnum);
- defer allocator.free(raw_section_header);
- const bytes_read = try in_file.preadAll(std.mem.sliceAsBytes(raw_section_header), header.shoff);
- if (bytes_read < @sizeOf(Elf_Phdr) * header.shnum)
- return error.TRUNCATED_ELF;
-
- for (section_header, raw_section_header) |*section, hdr| {
- section.* = .{ .section = hdr };
- }
- break :blk section_header;
- };
-
- // load data to memory for some sections:
- // string tables for access
- // sections than need modifications when other sections move.
- for (sections, 0..) |*section, idx| {
- const need_data = switch (section.section.sh_type) {
- elf.DT_VERSYM => true,
- elf.SHT_SYMTAB, elf.SHT_DYNSYM => true,
- else => false,
- };
- const need_strings = (idx == header.shstrndx);
-
- if (need_data or need_strings) {
- const buffer = try allocator.alignedAlloc(u8, section_memory_align, @intCast(section.section.sh_size));
- const bytes_read = try in_file.preadAll(buffer, section.section.sh_offset);
- if (bytes_read != section.section.sh_size) return error.TRUNCATED_ELF;
- section.payload = buffer;
- }
- }
-
- // fill-in sections info:
- // resolve the name
- // find if a program segment uses the section
- // categorize sections usage (used by program segments, debug datadase, common metadata, symbol table)
- for (sections) |*section| {
- section.segment = for (program_segments) |*seg| {
- if (sectionWithinSegment(section.section, seg.*)) break seg;
- } else null;
-
- if (section.section.sh_name != 0 and header.shstrndx != elf.SHN_UNDEF)
- section.name = std.mem.span(@as([*:0]const u8, @ptrCast(§ions[header.shstrndx].payload.?[section.section.sh_name])));
-
- const category_from_program: SectionCategory = if (section.segment != null) .exe else .debug;
- section.category = switch (section.section.sh_type) {
- elf.SHT_NOTE => .common,
- elf.SHT_SYMTAB => .symbols, // "strip all" vs "strip only debug"
- elf.SHT_DYNSYM => .exe,
- elf.SHT_PROGBITS => cat: {
- if (std.mem.eql(u8, section.name, ".comment")) break :cat .exe;
- if (std.mem.eql(u8, section.name, ".gnu_debuglink")) break :cat .none;
- break :cat category_from_program;
- },
- elf.SHT_LOPROC...elf.SHT_HIPROC => .common, // don't strip unknown sections
- elf.SHT_LOUSER...elf.SHT_HIUSER => .common, // don't strip unknown sections
- else => category_from_program,
- };
- }
-
- sections[0].category = .common; // mandatory null section
- if (header.shstrndx != elf.SHN_UNDEF)
- sections[header.shstrndx].category = .common; // string table for the headers
-
- // recursively propagate section categories to their linked sections, so that they are kept together
- var dirty: u1 = 1;
- while (dirty != 0) {
- dirty = 0;
-
- for (sections) |*section| {
- if (section.section.sh_link != elf.SHN_UNDEF)
- dirty |= ElfFileHelper.propagateCategory(§ions[section.section.sh_link].category, section.category);
- if ((section.section.sh_flags & elf.SHF_INFO_LINK) != 0 and section.section.sh_info != elf.SHN_UNDEF)
- dirty |= ElfFileHelper.propagateCategory(§ions[section.section.sh_info].category, section.category);
- }
- }
-
- return Self{
- .arena = arena,
- .raw_elf_header = raw_header,
- .program_segments = program_segments,
- .sections = sections,
- };
- }
-
- pub fn deinit(self: *Self) void {
- self.arena.deinit();
- }
-
- const Filter = ElfFileHelper.Filter;
- const DebugLink = ElfFileHelper.DebugLink;
- const EmitElfOptions = struct {
- section_filter: Filter = .all,
- debuglink: ?DebugLink = null,
- compress_debug: bool = false,
- add_section: ?AddSection = null,
- set_section_alignment: ?SetSectionAlignment = null,
- set_section_flags: ?SetSectionFlags = null,
- };
- fn emit(self: *const Self, gpa: Allocator, out_file: File, in_file: File, options: EmitElfOptions) !void {
- var arena = std.heap.ArenaAllocator.init(gpa);
- defer arena.deinit();
- const allocator = arena.allocator();
-
- // when emitting the stripped exe:
- // - unused sections are removed
- // when emitting the debug file:
- // - all sections are kept, but some are emptied and their types is changed to SHT_NOBITS
- // the program header is kept unchanged. (`strip` does update it, but `eu-strip` does not, and it still works)
-
- const Update = struct {
- action: ElfFileHelper.Action,
-
- // remap the indexs after omitting the filtered sections
- remap_idx: u16,
-
- // optionally overrides the payload from the source file
- payload: ?[]align(section_memory_align.toByteUnits()) const u8 = null,
- section: ?Elf_Shdr = null,
- };
- const sections_update = try allocator.alloc(Update, self.sections.len);
- const new_shnum = blk: {
- var next_idx: u16 = 0;
- for (self.sections, sections_update) |section, *update| {
- const action = ElfFileHelper.selectAction(section.category, options.section_filter);
- const remap_idx = idx: {
- if (action == .strip) break :idx elf.SHN_UNDEF;
- next_idx += 1;
- break :idx next_idx - 1;
- };
- update.* = Update{ .action = action, .remap_idx = remap_idx };
- }
-
- if (options.debuglink != null)
- next_idx += 1;
-
- if (options.add_section != null) {
- next_idx += 1;
- }
-
- break :blk next_idx;
- };
-
- // add a ".gnu_debuglink" to the string table if needed
- const debuglink_name: u32 = blk: {
- if (options.debuglink == null) break :blk elf.SHN_UNDEF;
- if (self.raw_elf_header.e_shstrndx == elf.SHN_UNDEF)
- fatal("zig objcopy: no strtab, cannot add the debuglink section", .{}); // TODO add the section if needed?
-
- const strtab = &self.sections[self.raw_elf_header.e_shstrndx];
- const update = §ions_update[self.raw_elf_header.e_shstrndx];
-
- const name: []const u8 = ".gnu_debuglink";
- const new_offset: u32 = @intCast(strtab.payload.?.len);
- const buf = try allocator.alignedAlloc(u8, section_memory_align, new_offset + name.len + 1);
- @memcpy(buf[0..new_offset], strtab.payload.?);
- @memcpy(buf[new_offset..][0..name.len], name);
- buf[new_offset + name.len] = 0;
-
- assert(update.action == .keep);
- update.payload = buf;
-
- break :blk new_offset;
- };
-
- // add user section to the string table if needed
- const user_section_name: u32 = blk: {
- if (options.add_section == null) break :blk elf.SHN_UNDEF;
- if (self.raw_elf_header.e_shstrndx == elf.SHN_UNDEF)
- fatal("zig objcopy: no strtab, cannot add the user section", .{}); // TODO add the section if needed?
-
- const strtab = &self.sections[self.raw_elf_header.e_shstrndx];
- const update = §ions_update[self.raw_elf_header.e_shstrndx];
-
- const name = options.add_section.?.section_name;
- const new_offset: u32 = @intCast(strtab.payload.?.len);
- const buf = try allocator.alignedAlloc(u8, section_memory_align, new_offset + name.len + 1);
- @memcpy(buf[0..new_offset], strtab.payload.?);
- @memcpy(buf[new_offset..][0..name.len], name);
- buf[new_offset + name.len] = 0;
-
- assert(update.action == .keep);
- update.payload = buf;
-
- break :blk new_offset;
- };
-
- // maybe compress .debug sections
- if (options.compress_debug) {
- for (self.sections[1..], sections_update[1..]) |section, *update| {
- if (update.action != .keep) continue;
- if (!std.mem.startsWith(u8, section.name, ".debug_")) continue;
- if ((section.section.sh_flags & elf.SHF_COMPRESSED) != 0) continue; // already compressed
-
- const chdr = Elf_Chdr{
- .ch_type = elf.COMPRESS.ZLIB,
- .ch_size = section.section.sh_size,
- .ch_addralign = section.section.sh_addralign,
- };
-
- const compressed_payload = try ElfFileHelper.tryCompressSection(allocator, in_file, section.section.sh_offset, section.section.sh_size, std.mem.asBytes(&chdr));
- if (compressed_payload) |payload| {
- update.payload = payload;
- update.section = section.section;
- update.section.?.sh_addralign = @alignOf(Elf_Chdr);
- update.section.?.sh_size = @intCast(payload.len);
- update.section.?.sh_flags |= elf.SHF_COMPRESSED;
- }
- }
- }
-
- var cmdbuf = std.ArrayList(ElfFileHelper.WriteCmd).init(allocator);
- defer cmdbuf.deinit();
- try cmdbuf.ensureUnusedCapacity(3 + new_shnum);
- var eof_offset: Elf_OffSize = 0; // track the end of the data written so far.
-
- // build the updated headers
- // nb: updated_elf_header will be updated before the actual write
- var updated_elf_header = self.raw_elf_header;
- if (updated_elf_header.e_shstrndx != elf.SHN_UNDEF)
- updated_elf_header.e_shstrndx = sections_update[updated_elf_header.e_shstrndx].remap_idx;
- cmdbuf.appendAssumeCapacity(.{ .write_data = .{ .data = std.mem.asBytes(&updated_elf_header), .out_offset = 0 } });
- eof_offset = @sizeOf(Elf_Ehdr);
-
- // program header as-is.
- // nb: for only-debug files, removing it appears to work, but is invalid by ELF specifcation.
- {
- assert(updated_elf_header.e_phoff == @sizeOf(Elf_Ehdr));
- const data = std.mem.sliceAsBytes(self.program_segments);
- assert(data.len == @as(usize, updated_elf_header.e_phentsize) * updated_elf_header.e_phnum);
- cmdbuf.appendAssumeCapacity(.{ .write_data = .{ .data = data, .out_offset = updated_elf_header.e_phoff } });
- eof_offset = updated_elf_header.e_phoff + @as(Elf_OffSize, @intCast(data.len));
- }
-
- // update sections and queue payload writes
- const updated_section_header = blk: {
- const dest_sections = try allocator.alloc(Elf_Shdr, new_shnum);
-
- {
- // the ELF format doesn't specify the order for all sections.
- // this code only supports when they are in increasing file order.
- var offset: u64 = eof_offset;
- for (self.sections[1..]) |section| {
- if (section.section.sh_type == elf.SHT_NOBITS)
- continue;
- if (section.section.sh_offset < offset) {
- fatal("zig objcopy: unsupported ELF file", .{});
- }
- offset = section.section.sh_offset;
- }
- }
-
- dest_sections[0] = self.sections[0].section;
-
- var dest_section_idx: u32 = 1;
- for (self.sections[1..], sections_update[1..]) |section, update| {
- if (update.action == .strip) continue;
- assert(update.remap_idx == dest_section_idx);
-
- const src = if (update.section) |*s| s else §ion.section;
- const dest = &dest_sections[dest_section_idx];
- const payload = if (update.payload) |data| data else section.payload;
- dest_section_idx += 1;
-
- dest.* = src.*;
-
- if (src.sh_link != elf.SHN_UNDEF)
- dest.sh_link = sections_update[src.sh_link].remap_idx;
- if ((src.sh_flags & elf.SHF_INFO_LINK) != 0 and src.sh_info != elf.SHN_UNDEF)
- dest.sh_info = sections_update[src.sh_info].remap_idx;
-
- if (payload) |data|
- dest.sh_size = @intCast(data.len);
-
- const addralign = if (src.sh_addralign == 0 or dest.sh_type == elf.SHT_NOBITS) 1 else src.sh_addralign;
- dest.sh_offset = std.mem.alignForward(Elf_OffSize, eof_offset, addralign);
- if (src.sh_offset != dest.sh_offset and section.segment != null and update.action != .empty and dest.sh_type != elf.SHT_NOTE and dest.sh_type != elf.SHT_NOBITS) {
- if (src.sh_offset > dest.sh_offset) {
- dest.sh_offset = src.sh_offset; // add padding to avoid modifing the program segments
- } else {
- fatal("zig objcopy: cannot adjust program segments", .{});
- }
- }
- assert(dest.sh_addr % addralign == dest.sh_offset % addralign);
-
- if (update.action == .empty)
- dest.sh_type = elf.SHT_NOBITS;
-
- if (dest.sh_type != elf.SHT_NOBITS) {
- if (payload) |src_data| {
- // update sections payload and write
- const dest_data = switch (src.sh_type) {
- elf.DT_VERSYM => dst_data: {
- const data = try allocator.alignedAlloc(u8, section_memory_align, src_data.len);
- @memcpy(data, src_data);
-
- const defs = @as([*]elf.Verdef, @ptrCast(data))[0 .. @as(usize, @intCast(src.sh_size)) / @sizeOf(elf.Verdef)];
- for (defs) |*def| switch (def.ndx) {
- .LOCAL, .GLOBAL => {},
- else => def.ndx = @enumFromInt(sections_update[src.sh_info].remap_idx),
- };
-
- break :dst_data data;
- },
- elf.SHT_SYMTAB, elf.SHT_DYNSYM => dst_data: {
- const data = try allocator.alignedAlloc(u8, section_memory_align, src_data.len);
- @memcpy(data, src_data);
-
- const syms = @as([*]Elf_Sym, @ptrCast(data))[0 .. @as(usize, @intCast(src.sh_size)) / @sizeOf(Elf_Sym)];
- for (syms) |*sym| {
- if (sym.st_shndx != elf.SHN_UNDEF and sym.st_shndx < elf.SHN_LORESERVE)
- sym.st_shndx = sections_update[sym.st_shndx].remap_idx;
- }
-
- break :dst_data data;
- },
- else => src_data,
- };
-
- assert(dest_data.len == dest.sh_size);
- cmdbuf.appendAssumeCapacity(.{ .write_data = .{ .data = dest_data, .out_offset = dest.sh_offset } });
- eof_offset = dest.sh_offset + dest.sh_size;
- } else {
- // direct contents copy
- cmdbuf.appendAssumeCapacity(.{ .copy_range = .{ .in_offset = src.sh_offset, .len = dest.sh_size, .out_offset = dest.sh_offset } });
- eof_offset = dest.sh_offset + dest.sh_size;
- }
- } else {
- // account for alignment padding even in empty sections to keep logical section order
- eof_offset = dest.sh_offset;
- }
- }
-
- // add a ".gnu_debuglink" section
- if (options.debuglink) |link| {
- const payload = payload: {
- const crc_offset = std.mem.alignForward(usize, link.name.len + 1, 4);
- const buf = try allocator.alignedAlloc(u8, .@"4", crc_offset + 4);
- @memcpy(buf[0..link.name.len], link.name);
- @memset(buf[link.name.len..crc_offset], 0);
- @memcpy(buf[crc_offset..], std.mem.asBytes(&link.crc32));
- break :payload buf;
- };
-
- dest_sections[dest_section_idx] = Elf_Shdr{
- .sh_name = debuglink_name,
- .sh_type = elf.SHT_PROGBITS,
- .sh_flags = 0,
- .sh_addr = 0,
- .sh_offset = eof_offset,
- .sh_size = @intCast(payload.len),
- .sh_link = elf.SHN_UNDEF,
- .sh_info = elf.SHN_UNDEF,
- .sh_addralign = 4,
- .sh_entsize = 0,
- };
- dest_section_idx += 1;
-
- cmdbuf.appendAssumeCapacity(.{ .write_data = .{ .data = payload, .out_offset = eof_offset } });
- eof_offset += @as(Elf_OffSize, @intCast(payload.len));
- }
-
- // --add-section
- if (options.add_section) |add_section| {
- var section_file = fs.cwd().openFile(add_section.file_path, .{}) catch |err|
- fatal("unable to open '{s}': {s}", .{ add_section.file_path, @errorName(err) });
- defer section_file.close();
-
- const payload = try section_file.readToEndAlloc(arena.allocator(), std.math.maxInt(usize));
-
- dest_sections[dest_section_idx] = Elf_Shdr{
- .sh_name = user_section_name,
- .sh_type = elf.SHT_PROGBITS,
- .sh_flags = 0,
- .sh_addr = 0,
- .sh_offset = eof_offset,
- .sh_size = @intCast(payload.len),
- .sh_link = elf.SHN_UNDEF,
- .sh_info = elf.SHN_UNDEF,
- .sh_addralign = 4,
- .sh_entsize = 0,
- };
- dest_section_idx += 1;
-
- cmdbuf.appendAssumeCapacity(.{ .write_data = .{ .data = payload, .out_offset = eof_offset } });
- eof_offset += @as(Elf_OffSize, @intCast(payload.len));
- }
-
- assert(dest_section_idx == new_shnum);
- break :blk dest_sections;
- };
-
- // --set-section-alignment: overwrite alignment
- if (options.set_section_alignment) |set_align| {
- if (self.raw_elf_header.e_shstrndx == elf.SHN_UNDEF)
- fatal("zig objcopy: no strtab, cannot add the user section", .{}); // TODO add the section if needed?
-
- const strtab = §ions_update[self.raw_elf_header.e_shstrndx];
- for (updated_section_header) |*section| {
- const section_name = std.mem.span(@as([*:0]const u8, @ptrCast(&strtab.payload.?[section.sh_name])));
- if (std.mem.eql(u8, section_name, set_align.section_name)) {
- section.sh_addralign = set_align.alignment;
- break;
- }
- } else std.log.warn("Skipping --set-section-alignment. Section '{s}' not found", .{set_align.section_name});
- }
-
- // --set-section-flags: overwrite flags
- if (options.set_section_flags) |set_flags| {
- if (self.raw_elf_header.e_shstrndx == elf.SHN_UNDEF)
- fatal("zig objcopy: no strtab, cannot add the user section", .{}); // TODO add the section if needed?
-
- const strtab = §ions_update[self.raw_elf_header.e_shstrndx];
- for (updated_section_header) |*section| {
- const section_name = std.mem.span(@as([*:0]const u8, @ptrCast(&strtab.payload.?[section.sh_name])));
- if (std.mem.eql(u8, section_name, set_flags.section_name)) {
- section.sh_flags = std.elf.SHF_WRITE; // default is writable cleared by "readonly"
- const f = set_flags.flags;
-
- // Supporting a subset of GNU and LLVM objcopy for ELF only
- // GNU:
- // alloc: add SHF_ALLOC
- // contents: if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing
- // load: if section is SHT_NOBITS, set SHT_PROGBITS, otherwise do nothing (same as contents)
- // noload: not ELF relevant
- // readonly: clear default SHF_WRITE flag
- // code: add SHF_EXECINSTR
- // data: not ELF relevant
- // rom: ignored
- // exclude: add SHF_EXCLUDE
- // share: not ELF relevant
- // debug: not ELF relevant
- // large: add SHF_X86_64_LARGE. Fatal error if target is not x86_64
- if (f.alloc) section.sh_flags |= std.elf.SHF_ALLOC;
- if (f.contents or f.load) {
- if (section.sh_type == std.elf.SHT_NOBITS) section.sh_type = std.elf.SHT_PROGBITS;
- }
- if (f.readonly) section.sh_flags &= ~@as(@TypeOf(section.sh_type), std.elf.SHF_WRITE);
- if (f.code) section.sh_flags |= std.elf.SHF_EXECINSTR;
- if (f.exclude) section.sh_flags |= std.elf.SHF_EXCLUDE;
- if (f.large) {
- if (updated_elf_header.e_machine != std.elf.EM.X86_64)
- fatal("zig objcopy: 'large' section flag is only supported on x86_64 targets", .{});
- section.sh_flags |= std.elf.SHF_X86_64_LARGE;
- }
-
- // LLVM:
- // merge: add SHF_MERGE
- // strings: add SHF_STRINGS
- if (f.merge) section.sh_flags |= std.elf.SHF_MERGE;
- if (f.strings) section.sh_flags |= std.elf.SHF_STRINGS;
- break;
- }
- } else std.log.warn("Skipping --set-section-flags. Section '{s}' not found", .{set_flags.section_name});
- }
-
- // write the section header at the tail
- {
- const offset = std.mem.alignForward(Elf_OffSize, eof_offset, @alignOf(Elf_Shdr));
-
- const data = std.mem.sliceAsBytes(updated_section_header);
- assert(data.len == @as(usize, updated_elf_header.e_shentsize) * new_shnum);
- updated_elf_header.e_shoff = offset;
- updated_elf_header.e_shnum = new_shnum;
-
- cmdbuf.appendAssumeCapacity(.{ .write_data = .{ .data = data, .out_offset = updated_elf_header.e_shoff } });
- }
-
- try ElfFileHelper.write(allocator, out_file, in_file, cmdbuf.items);
- }
-
- fn sectionWithinSegment(section: Elf_Shdr, segment: Elf_Phdr) bool {
- const file_size = if (section.sh_type == elf.SHT_NOBITS) 0 else section.sh_size;
- return segment.p_offset <= section.sh_offset and (segment.p_offset + segment.p_filesz) >= (section.sh_offset + file_size);
- }
- };
-}
-
-const ElfFileHelper = struct {
- const DebugLink = struct { name: []const u8, crc32: u32 };
- const Filter = enum { all, program, debug, program_and_symbols, debug_and_symbols };
-
- const SectionCategory = enum { common, exe, debug, symbols, none };
- fn propagateCategory(cur: *SectionCategory, new: SectionCategory) u1 {
- const cat: SectionCategory = switch (cur.*) {
- .none => new,
- .common => .common,
- .debug => switch (new) {
- .none, .debug => .debug,
- else => new,
- },
- .exe => switch (new) {
- .common => .common,
- .none, .debug, .exe => .exe,
- .symbols => .exe,
- },
- .symbols => switch (new) {
- .none, .common, .debug, .exe => unreachable,
- .symbols => .symbols,
- },
- };
-
- if (cur.* != cat) {
- cur.* = cat;
- return 1;
- } else {
- return 0;
- }
- }
-
- const Action = enum { keep, strip, empty };
- fn selectAction(category: SectionCategory, filter: Filter) Action {
- if (category == .none) return .strip;
- return switch (filter) {
- .all => switch (category) {
- .none => .strip,
- else => .keep,
- },
- .program => switch (category) {
- .common, .exe => .keep,
- else => .strip,
- },
- .program_and_symbols => switch (category) {
- .common, .exe, .symbols => .keep,
- else => .strip,
- },
- .debug => switch (category) {
- .exe, .symbols => .empty,
- .none => .strip,
- else => .keep,
- },
- .debug_and_symbols => switch (category) {
- .exe => .empty,
- .none => .strip,
- else => .keep,
- },
- };
- }
-
- const WriteCmd = union(enum) {
- copy_range: struct { in_offset: u64, len: u64, out_offset: u64 },
- write_data: struct { data: []const u8, out_offset: u64 },
- };
- fn write(allocator: Allocator, out_file: File, in_file: File, cmds: []const WriteCmd) !void {
- // consolidate holes between writes:
- // by coping original padding data from in_file (by fusing contiguous ranges)
- // by writing zeroes otherwise
- const zeroes = [1]u8{0} ** 4096;
- var consolidated = std.ArrayList(WriteCmd).init(allocator);
- defer consolidated.deinit();
- try consolidated.ensureUnusedCapacity(cmds.len * 2);
- var offset: u64 = 0;
- var fused_cmd: ?WriteCmd = null;
- for (cmds) |cmd| {
- switch (cmd) {
- .write_data => |data| {
- assert(data.out_offset >= offset);
- if (fused_cmd) |prev| {
- consolidated.appendAssumeCapacity(prev);
- fused_cmd = null;
- }
- if (data.out_offset > offset) {
- consolidated.appendAssumeCapacity(.{ .write_data = .{ .data = zeroes[0..@intCast(data.out_offset - offset)], .out_offset = offset } });
- }
- consolidated.appendAssumeCapacity(cmd);
- offset = data.out_offset + data.data.len;
- },
- .copy_range => |range| {
- assert(range.out_offset >= offset);
- if (fused_cmd) |prev| {
- if (range.in_offset >= prev.copy_range.in_offset + prev.copy_range.len and (range.out_offset - prev.copy_range.out_offset == range.in_offset - prev.copy_range.in_offset)) {
- fused_cmd = .{ .copy_range = .{
- .in_offset = prev.copy_range.in_offset,
- .out_offset = prev.copy_range.out_offset,
- .len = (range.out_offset + range.len) - prev.copy_range.out_offset,
- } };
- } else {
- consolidated.appendAssumeCapacity(prev);
- if (range.out_offset > offset) {
- consolidated.appendAssumeCapacity(.{ .write_data = .{ .data = zeroes[0..@intCast(range.out_offset - offset)], .out_offset = offset } });
- }
- fused_cmd = cmd;
- }
- } else {
- fused_cmd = cmd;
- }
- offset = range.out_offset + range.len;
- },
- }
- }
- if (fused_cmd) |cmd| {
- consolidated.appendAssumeCapacity(cmd);
- }
-
- // write the output file
- for (consolidated.items) |cmd| {
- switch (cmd) {
- .write_data => |data| {
- var iovec = [_]std.posix.iovec_const{.{ .base = data.data.ptr, .len = data.data.len }};
- try out_file.pwritevAll(&iovec, data.out_offset);
- },
- .copy_range => |range| {
- const copied_bytes = try in_file.copyRangeAll(range.in_offset, out_file, range.out_offset, range.len);
- if (copied_bytes < range.len) return error.TRUNCATED_ELF;
- },
- }
- }
- }
-
- fn tryCompressSection(allocator: Allocator, in_file: File, offset: u64, size: u64, prefix: []const u8) !?[]align(8) const u8 {
- if (size < prefix.len) return null;
-
- try in_file.seekTo(offset);
- var section_reader = std.io.limitedReader(in_file.deprecatedReader(), size);
-
- // allocate as large as decompressed data. if the compression doesn't fit, keep the data uncompressed.
- const compressed_data = try allocator.alignedAlloc(u8, .@"8", @intCast(size));
- var compressed_stream = std.io.fixedBufferStream(compressed_data);
-
- try compressed_stream.writer().writeAll(prefix);
-
- {
- var compressor = try std.compress.zlib.compressor(compressed_stream.writer(), .{});
-
- var buf: [8000]u8 = undefined;
- while (true) {
- const bytes_read = try section_reader.read(&buf);
- if (bytes_read == 0) break;
- const bytes_written = compressor.write(buf[0..bytes_read]) catch |err| switch (err) {
- error.NoSpaceLeft => {
- allocator.free(compressed_data);
- return null;
- },
- else => return err,
- };
- std.debug.assert(bytes_written == bytes_read);
- }
- compressor.finish() catch |err| switch (err) {
- error.NoSpaceLeft => {
- allocator.free(compressed_data);
- return null;
- },
- else => return err,
- };
- }
-
- const compressed_len: usize = @intCast(compressed_stream.getPos() catch unreachable);
- const data = allocator.realloc(compressed_data, compressed_len) catch compressed_data;
- return data[0..compressed_len];
- }
-
- fn createDebugLink(path: []const u8) DebugLink {
- const file = std.fs.cwd().openFile(path, .{}) catch |err| {
- fatal("zig objcopy: could not open `{s}`: {s}\n", .{ path, @errorName(err) });
- };
- defer file.close();
-
- const crc = ElfFileHelper.computeFileCrc(file) catch |err| {
- fatal("zig objcopy: could not read `{s}`: {s}\n", .{ path, @errorName(err) });
- };
- return .{
- .name = std.fs.path.basename(path),
- .crc32 = crc,
- };
- }
-
- fn computeFileCrc(file: File) !u32 {
- var buf: [8000]u8 = undefined;
-
- try file.seekTo(0);
- var hasher = std.hash.Crc32.init();
- while (true) {
- const bytes_read = try file.read(&buf);
- if (bytes_read == 0) break;
- hasher.update(buf[0..bytes_read]);
- }
- return hasher.final();
- }
-};
-
const SectionFlags = packed struct {
alloc: bool = false,
contents: bool = false,
diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig
@@ -169,7 +169,7 @@ pub const Output = struct {
pub fn create(owner: *std.Build, name: []const u8) *Run {
const run = owner.allocator.create(Run) catch @panic("OOM");
run.* = .{
- .step = Step.init(.{
+ .step = .init(.{
.id = base_id,
.name = name,
.owner = owner,
@@ -1769,13 +1769,22 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
child.stdin = null;
},
.lazy_path => |lazy_path| {
- const path = lazy_path.getPath2(b, &run.step);
- const file = b.build_root.handle.openFile(path, .{}) catch |err| {
+ const path = lazy_path.getPath3(b, &run.step);
+ const file = path.root_dir.handle.openFile(path.subPathOrDot(), .{}) catch |err| {
return run.step.fail("unable to open stdin file: {s}", .{@errorName(err)});
};
defer file.close();
- child.stdin.?.writeFileAll(file, .{}) catch |err| {
- return run.step.fail("unable to write file to stdin: {s}", .{@errorName(err)});
+ // TODO https://github.com/ziglang/zig/issues/23955
+ var buffer: [1024]u8 = undefined;
+ var file_reader = file.reader(&buffer);
+ var stdin_writer = child.stdin.?.writer(&.{});
+ _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) {
+ error.ReadFailed => return run.step.fail("failed to read from {f}: {t}", .{
+ path, file_reader.err.?,
+ }),
+ error.WriteFailed => return run.step.fail("failed to write to stdin: {t}", .{
+ stdin_writer.err.?,
+ }),
};
child.stdin.?.close();
child.stdin = null;
diff --git a/lib/std/c.zig b/lib/std/c.zig
@@ -10497,9 +10497,9 @@ pub const sysconf = switch (native_os) {
pub const sf_hdtr = switch (native_os) {
.freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct {
- headers: [*]const iovec_const,
+ headers: ?[*]const iovec_const,
hdr_cnt: c_int,
- trailers: [*]const iovec_const,
+ trailers: ?[*]const iovec_const,
trl_cnt: c_int,
},
else => void,
diff --git a/lib/std/elf.zig b/lib/std/elf.zig
@@ -482,6 +482,7 @@ pub const Header = struct {
is_64: bool,
endian: std.builtin.Endian,
os_abi: OSABI,
+ /// The meaning of this value depends on `os_abi`.
abi_version: u8,
type: ET,
machine: EM,
@@ -494,205 +495,135 @@ pub const Header = struct {
shnum: u16,
shstrndx: u16,
- pub fn program_header_iterator(self: Header, parse_source: anytype) ProgramHeaderIterator(@TypeOf(parse_source)) {
- return ProgramHeaderIterator(@TypeOf(parse_source)){
- .elf_header = self,
- .parse_source = parse_source,
+ pub fn iterateProgramHeaders(h: Header, file_reader: *std.fs.File.Reader) ProgramHeaderIterator {
+ return .{
+ .elf_header = h,
+ .file_reader = file_reader,
};
}
- pub fn section_header_iterator(self: Header, parse_source: anytype) SectionHeaderIterator(@TypeOf(parse_source)) {
- return SectionHeaderIterator(@TypeOf(parse_source)){
- .elf_header = self,
- .parse_source = parse_source,
+ pub fn iterateSectionHeaders(h: Header, file_reader: *std.fs.File.Reader) SectionHeaderIterator {
+ return .{
+ .elf_header = h,
+ .file_reader = file_reader,
};
}
- pub fn read(parse_source: anytype) !Header {
- var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined;
- try parse_source.seekableStream().seekTo(0);
- try parse_source.deprecatedReader().readNoEof(&hdr_buf);
- return Header.parse(&hdr_buf);
- }
+ pub const ReadError = std.Io.Reader.Error || error{
+ InvalidElfMagic,
+ InvalidElfVersion,
+ InvalidElfClass,
+ InvalidElfEndian,
+ };
- pub fn parse(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) !Header {
- const hdr32 = @as(*const Elf32_Ehdr, @ptrCast(hdr_buf));
- const hdr64 = @as(*const Elf64_Ehdr, @ptrCast(hdr_buf));
- if (!mem.eql(u8, hdr32.e_ident[0..4], MAGIC)) return error.InvalidElfMagic;
- if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion;
+ pub fn read(r: *std.Io.Reader) ReadError!Header {
+ const buf = try r.peek(@sizeOf(Elf64_Ehdr));
- const is_64 = switch (hdr32.e_ident[EI_CLASS]) {
- ELFCLASS32 => false,
- ELFCLASS64 => true,
- else => return error.InvalidElfClass,
- };
+ if (!mem.eql(u8, buf[0..4], MAGIC)) return error.InvalidElfMagic;
+ if (buf[EI_VERSION] != 1) return error.InvalidElfVersion;
- const endian: std.builtin.Endian = switch (hdr32.e_ident[EI_DATA]) {
+ const endian: std.builtin.Endian = switch (buf[EI_DATA]) {
ELFDATA2LSB => .little,
ELFDATA2MSB => .big,
else => return error.InvalidElfEndian,
};
- const need_bswap = endian != native_endian;
+ return switch (buf[EI_CLASS]) {
+ ELFCLASS32 => .init(try r.takeStruct(Elf32_Ehdr, endian), endian),
+ ELFCLASS64 => .init(try r.takeStruct(Elf64_Ehdr, endian), endian),
+ else => return error.InvalidElfClass,
+ };
+ }
+
+ pub fn init(hdr: anytype, endian: std.builtin.Endian) Header {
// Converting integers to exhaustive enums using `@enumFromInt` could cause a panic.
comptime assert(!@typeInfo(OSABI).@"enum".is_exhaustive);
- const os_abi: OSABI = @enumFromInt(hdr32.e_ident[EI_OSABI]);
+ return .{
+ .is_64 = switch (@TypeOf(hdr)) {
+ Elf32_Ehdr => false,
+ Elf64_Ehdr => true,
+ else => @compileError("bad type"),
+ },
+ .endian = endian,
+ .os_abi = @enumFromInt(hdr.e_ident[EI_OSABI]),
+ .abi_version = hdr.e_ident[EI_ABIVERSION],
+ .type = hdr.e_type,
+ .machine = hdr.e_machine,
+ .entry = hdr.e_entry,
+ .phoff = hdr.e_phoff,
+ .shoff = hdr.e_shoff,
+ .phentsize = hdr.e_phentsize,
+ .phnum = hdr.e_phnum,
+ .shentsize = hdr.e_shentsize,
+ .shnum = hdr.e_shnum,
+ .shstrndx = hdr.e_shstrndx,
+ };
+ }
+};
- // The meaning of this value depends on `os_abi` so just make it available as `u8`.
- const abi_version = hdr32.e_ident[EI_ABIVERSION];
+pub const ProgramHeaderIterator = struct {
+ elf_header: Header,
+ file_reader: *std.fs.File.Reader,
+ index: usize = 0,
- const @"type" = if (need_bswap) blk: {
- comptime assert(!@typeInfo(ET).@"enum".is_exhaustive);
- const value = @intFromEnum(hdr32.e_type);
- break :blk @as(ET, @enumFromInt(@byteSwap(value)));
- } else hdr32.e_type;
+ pub fn next(it: *ProgramHeaderIterator) !?Elf64_Phdr {
+ if (it.index >= it.elf_header.phnum) return null;
+ defer it.index += 1;
- const machine = if (need_bswap) blk: {
- comptime assert(!@typeInfo(EM).@"enum".is_exhaustive);
- const value = @intFromEnum(hdr32.e_machine);
- break :blk @as(EM, @enumFromInt(@byteSwap(value)));
- } else hdr32.e_machine;
+ if (it.elf_header.is_64) {
+ const offset = it.elf_header.phoff + @sizeOf(Elf64_Phdr) * it.index;
+ try it.file_reader.seekTo(offset);
+ const phdr = try it.file_reader.interface.takeStruct(Elf64_Phdr, it.elf_header.endian);
+ return phdr;
+ }
- return @as(Header, .{
- .is_64 = is_64,
- .endian = endian,
- .os_abi = os_abi,
- .abi_version = abi_version,
- .type = @"type",
- .machine = machine,
- .entry = int(is_64, need_bswap, hdr32.e_entry, hdr64.e_entry),
- .phoff = int(is_64, need_bswap, hdr32.e_phoff, hdr64.e_phoff),
- .shoff = int(is_64, need_bswap, hdr32.e_shoff, hdr64.e_shoff),
- .phentsize = int(is_64, need_bswap, hdr32.e_phentsize, hdr64.e_phentsize),
- .phnum = int(is_64, need_bswap, hdr32.e_phnum, hdr64.e_phnum),
- .shentsize = int(is_64, need_bswap, hdr32.e_shentsize, hdr64.e_shentsize),
- .shnum = int(is_64, need_bswap, hdr32.e_shnum, hdr64.e_shnum),
- .shstrndx = int(is_64, need_bswap, hdr32.e_shstrndx, hdr64.e_shstrndx),
- });
+ const offset = it.elf_header.phoff + @sizeOf(Elf32_Phdr) * it.index;
+ try it.file_reader.seekTo(offset);
+ const phdr = try it.file_reader.interface.takeStruct(Elf32_Phdr, it.elf_header.endian);
+ return .{
+ .p_type = phdr.p_type,
+ .p_offset = phdr.p_offset,
+ .p_vaddr = phdr.p_vaddr,
+ .p_paddr = phdr.p_paddr,
+ .p_filesz = phdr.p_filesz,
+ .p_memsz = phdr.p_memsz,
+ .p_flags = phdr.p_flags,
+ .p_align = phdr.p_align,
+ };
}
};
-pub fn ProgramHeaderIterator(comptime ParseSource: anytype) type {
- return struct {
- elf_header: Header,
- parse_source: ParseSource,
- index: usize = 0,
-
- pub fn next(self: *@This()) !?Elf64_Phdr {
- if (self.index >= self.elf_header.phnum) return null;
- defer self.index += 1;
-
- if (self.elf_header.is_64) {
- var phdr: Elf64_Phdr = undefined;
- const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index;
- try self.parse_source.seekableStream().seekTo(offset);
- try self.parse_source.deprecatedReader().readNoEof(mem.asBytes(&phdr));
-
- // ELF endianness matches native endianness.
- if (self.elf_header.endian == native_endian) return phdr;
-
- // Convert fields to native endianness.
- mem.byteSwapAllFields(Elf64_Phdr, &phdr);
- return phdr;
- }
-
- var phdr: Elf32_Phdr = undefined;
- const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index;
- try self.parse_source.seekableStream().seekTo(offset);
- try self.parse_source.deprecatedReader().readNoEof(mem.asBytes(&phdr));
-
- // ELF endianness does NOT match native endianness.
- if (self.elf_header.endian != native_endian) {
- // Convert fields to native endianness.
- mem.byteSwapAllFields(Elf32_Phdr, &phdr);
- }
-
- // Convert 32-bit header to 64-bit.
- return Elf64_Phdr{
- .p_type = phdr.p_type,
- .p_offset = phdr.p_offset,
- .p_vaddr = phdr.p_vaddr,
- .p_paddr = phdr.p_paddr,
- .p_filesz = phdr.p_filesz,
- .p_memsz = phdr.p_memsz,
- .p_flags = phdr.p_flags,
- .p_align = phdr.p_align,
- };
- }
- };
-}
+pub const SectionHeaderIterator = struct {
+ elf_header: Header,
+ file_reader: *std.fs.File.Reader,
+ index: usize = 0,
-pub fn SectionHeaderIterator(comptime ParseSource: anytype) type {
- return struct {
- elf_header: Header,
- parse_source: ParseSource,
- index: usize = 0,
-
- pub fn next(self: *@This()) !?Elf64_Shdr {
- if (self.index >= self.elf_header.shnum) return null;
- defer self.index += 1;
-
- if (self.elf_header.is_64) {
- var shdr: Elf64_Shdr = undefined;
- const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index;
- try self.parse_source.seekableStream().seekTo(offset);
- try self.parse_source.deprecatedReader().readNoEof(mem.asBytes(&shdr));
-
- // ELF endianness matches native endianness.
- if (self.elf_header.endian == native_endian) return shdr;
-
- // Convert fields to native endianness.
- mem.byteSwapAllFields(Elf64_Shdr, &shdr);
- return shdr;
- }
-
- var shdr: Elf32_Shdr = undefined;
- const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index;
- try self.parse_source.seekableStream().seekTo(offset);
- try self.parse_source.deprecatedReader().readNoEof(mem.asBytes(&shdr));
-
- // ELF endianness does NOT match native endianness.
- if (self.elf_header.endian != native_endian) {
- // Convert fields to native endianness.
- mem.byteSwapAllFields(Elf32_Shdr, &shdr);
- }
-
- // Convert 32-bit header to 64-bit.
- return Elf64_Shdr{
- .sh_name = shdr.sh_name,
- .sh_type = shdr.sh_type,
- .sh_flags = shdr.sh_flags,
- .sh_addr = shdr.sh_addr,
- .sh_offset = shdr.sh_offset,
- .sh_size = shdr.sh_size,
- .sh_link = shdr.sh_link,
- .sh_info = shdr.sh_info,
- .sh_addralign = shdr.sh_addralign,
- .sh_entsize = shdr.sh_entsize,
- };
- }
- };
-}
+ pub fn next(it: *SectionHeaderIterator) !?Elf64_Shdr {
+ if (it.index >= it.elf_header.shnum) return null;
+ defer it.index += 1;
-fn int(is_64: bool, need_bswap: bool, int_32: anytype, int_64: anytype) @TypeOf(int_64) {
- if (is_64) {
- if (need_bswap) {
- return @byteSwap(int_64);
- } else {
- return int_64;
+ if (it.elf_header.is_64) {
+ try it.file_reader.seekTo(it.elf_header.shoff + @sizeOf(Elf64_Shdr) * it.index);
+ const shdr = try it.file_reader.interface.takeStruct(Elf64_Shdr, it.elf_header.endian);
+ return shdr;
}
- } else {
- return int32(need_bswap, int_32, @TypeOf(int_64));
- }
-}
-fn int32(need_bswap: bool, int_32: anytype, comptime Int64: anytype) Int64 {
- if (need_bswap) {
- return @byteSwap(int_32);
- } else {
- return int_32;
+ try it.file_reader.seekTo(it.elf_header.shoff + @sizeOf(Elf32_Shdr) * it.index);
+ const shdr = try it.file_reader.interface.takeStruct(Elf32_Shdr, it.elf_header.endian);
+ return .{
+ .sh_name = shdr.sh_name,
+ .sh_type = shdr.sh_type,
+ .sh_flags = shdr.sh_flags,
+ .sh_addr = shdr.sh_addr,
+ .sh_offset = shdr.sh_offset,
+ .sh_size = shdr.sh_size,
+ .sh_link = shdr.sh_link,
+ .sh_info = shdr.sh_info,
+ .sh_addralign = shdr.sh_addralign,
+ .sh_entsize = shdr.sh_entsize,
+ };
}
-}
+};
pub const ELFCLASSNONE = 0;
pub const ELFCLASS32 = 1;
diff --git a/lib/std/fs/AtomicFile.zig b/lib/std/fs/AtomicFile.zig
@@ -1,6 +1,13 @@
-file: File,
-// TODO either replace this with rand_buf or use []u16 on Windows
-tmp_path_buf: [tmp_path_len:0]u8,
+const AtomicFile = @This();
+const std = @import("../std.zig");
+const File = std.fs.File;
+const Dir = std.fs.Dir;
+const fs = std.fs;
+const assert = std.debug.assert;
+const posix = std.posix;
+
+file_writer: File.Writer,
+random_integer: u64,
dest_basename: []const u8,
file_open: bool,
file_exists: bool,
@@ -9,35 +16,24 @@ dir: Dir,
pub const InitError = File.OpenError;
-pub const random_bytes_len = 12;
-const tmp_path_len = fs.base64_encoder.calcSize(random_bytes_len);
-
/// Note that the `Dir.atomicFile` API may be more handy than this lower-level function.
pub fn init(
dest_basename: []const u8,
mode: File.Mode,
dir: Dir,
close_dir_on_deinit: bool,
+ write_buffer: []u8,
) InitError!AtomicFile {
- var rand_buf: [random_bytes_len]u8 = undefined;
- var tmp_path_buf: [tmp_path_len:0]u8 = undefined;
-
while (true) {
- std.crypto.random.bytes(rand_buf[0..]);
- const tmp_path = fs.base64_encoder.encode(&tmp_path_buf, &rand_buf);
- tmp_path_buf[tmp_path.len] = 0;
-
- const file = dir.createFile(
- tmp_path,
- .{ .mode = mode, .exclusive = true },
- ) catch |err| switch (err) {
+ const random_integer = std.crypto.random.int(u64);
+ const tmp_sub_path = std.fmt.hex(random_integer);
+ const file = dir.createFile(&tmp_sub_path, .{ .mode = mode, .exclusive = true }) catch |err| switch (err) {
error.PathAlreadyExists => continue,
else => |e| return e,
};
-
- return AtomicFile{
- .file = file,
- .tmp_path_buf = tmp_path_buf,
+ return .{
+ .file_writer = file.writer(write_buffer),
+ .random_integer = random_integer,
.dest_basename = dest_basename,
.file_open = true,
.file_exists = true,
@@ -48,41 +44,51 @@ pub fn init(
}
/// Always call deinit, even after a successful finish().
-pub fn deinit(self: *AtomicFile) void {
- if (self.file_open) {
- self.file.close();
- self.file_open = false;
+pub fn deinit(af: *AtomicFile) void {
+ if (af.file_open) {
+ af.file_writer.file.close();
+ af.file_open = false;
}
- if (self.file_exists) {
- self.dir.deleteFile(&self.tmp_path_buf) catch {};
- self.file_exists = false;
+ if (af.file_exists) {
+ const tmp_sub_path = std.fmt.hex(af.random_integer);
+ af.dir.deleteFile(&tmp_sub_path) catch {};
+ af.file_exists = false;
}
- if (self.close_dir_on_deinit) {
- self.dir.close();
+ if (af.close_dir_on_deinit) {
+ af.dir.close();
}
- self.* = undefined;
+ af.* = undefined;
}
-pub const FinishError = posix.RenameError;
+pub const FlushError = File.WriteError;
+
+pub fn flush(af: *AtomicFile) FlushError!void {
+ af.file_writer.interface.flush() catch |err| switch (err) {
+ error.WriteFailed => return af.file_writer.err.?,
+ };
+}
+
+pub const RenameIntoPlaceError = posix.RenameError;
/// On Windows, this function introduces a period of time where some file
/// system operations on the destination file will result in
/// `error.AccessDenied`, including rename operations (such as the one used in
/// this function).
-pub fn finish(self: *AtomicFile) FinishError!void {
- assert(self.file_exists);
- if (self.file_open) {
- self.file.close();
- self.file_open = false;
+pub fn renameIntoPlace(af: *AtomicFile) RenameIntoPlaceError!void {
+ assert(af.file_exists);
+ if (af.file_open) {
+ af.file_writer.file.close();
+ af.file_open = false;
}
- try posix.renameat(self.dir.fd, self.tmp_path_buf[0..], self.dir.fd, self.dest_basename);
- self.file_exists = false;
+ const tmp_sub_path = std.fmt.hex(af.random_integer);
+ try posix.renameat(af.dir.fd, &tmp_sub_path, af.dir.fd, af.dest_basename);
+ af.file_exists = false;
}
-const AtomicFile = @This();
-const std = @import("../std.zig");
-const File = std.fs.File;
-const Dir = std.fs.Dir;
-const fs = std.fs;
-const assert = std.debug.assert;
-const posix = std.posix;
+pub const FinishError = FlushError || RenameIntoPlaceError;
+
+/// Combination of `flush` followed by `renameIntoPlace`.
+pub fn finish(af: *AtomicFile) FinishError!void {
+ try af.flush();
+ try af.renameIntoPlace();
+}
diff --git a/lib/std/fs/Dir.zig b/lib/std/fs/Dir.zig
@@ -1,3 +1,20 @@
+const Dir = @This();
+const builtin = @import("builtin");
+const std = @import("../std.zig");
+const File = std.fs.File;
+const AtomicFile = std.fs.AtomicFile;
+const base64_encoder = fs.base64_encoder;
+const posix = std.posix;
+const mem = std.mem;
+const path = fs.path;
+const fs = std.fs;
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const linux = std.os.linux;
+const windows = std.os.windows;
+const native_os = builtin.os.tag;
+const have_flock = @TypeOf(posix.system.flock) != void;
+
fd: Handle,
pub const Handle = posix.fd_t;
@@ -1862,9 +1879,10 @@ pub fn symLinkW(
/// Same as `symLink`, except tries to create the symbolic link until it
/// succeeds or encounters an error other than `error.PathAlreadyExists`.
-/// On Windows, both paths should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/).
-/// On WASI, both paths should be encoded as valid UTF-8.
-/// On other platforms, both paths are an opaque sequence of bytes with no particular encoding.
+///
+/// * On Windows, both paths should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/).
+/// * On WASI, both paths should be encoded as valid UTF-8.
+/// * On other platforms, both paths are an opaque sequence of bytes with no particular encoding.
pub fn atomicSymLink(
dir: Dir,
target_path: []const u8,
@@ -1880,9 +1898,8 @@ pub fn atomicSymLink(
const dirname = path.dirname(sym_link_path) orelse ".";
- var rand_buf: [AtomicFile.random_bytes_len]u8 = undefined;
-
- const temp_path_len = dirname.len + 1 + base64_encoder.calcSize(rand_buf.len);
+ const rand_len = @sizeOf(u64) * 2;
+ const temp_path_len = dirname.len + 1 + rand_len;
var temp_path_buf: [fs.max_path_bytes]u8 = undefined;
if (temp_path_len > temp_path_buf.len) return error.NameTooLong;
@@ -1892,8 +1909,8 @@ pub fn atomicSymLink(
const temp_path = temp_path_buf[0..temp_path_len];
while (true) {
- crypto.random.bytes(rand_buf[0..]);
- _ = base64_encoder.encode(temp_path[dirname.len + 1 ..], rand_buf[0..]);
+ const random_integer = std.crypto.random.int(u64);
+ temp_path[dirname.len + 1 ..][0..rand_len].* = std.fmt.hex(random_integer);
if (dir.symLink(target_path, temp_path, flags)) {
return dir.rename(temp_path, sym_link_path);
@@ -2552,25 +2569,42 @@ pub fn updateFile(
try dest_dir.makePath(dirname);
}
- var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = actual_mode });
+ var buffer: [1000]u8 = undefined; // Used only when direct fd-to-fd is not available.
+ var atomic_file = try dest_dir.atomicFile(dest_path, .{
+ .mode = actual_mode,
+ .write_buffer = &buffer,
+ });
defer atomic_file.deinit();
- try atomic_file.file.writeFileAll(src_file, .{ .in_len = src_stat.size });
- try atomic_file.file.updateTimes(src_stat.atime, src_stat.mtime);
+ var src_reader: File.Reader = .initSize(src_file, &.{}, src_stat.size);
+ const dest_writer = &atomic_file.file_writer.interface;
+
+ _ = dest_writer.sendFileAll(&src_reader, .unlimited) catch |err| switch (err) {
+ error.ReadFailed => return src_reader.err.?,
+ error.WriteFailed => return atomic_file.file_writer.err.?,
+ };
+ try atomic_file.file_writer.file.updateTimes(src_stat.atime, src_stat.mtime);
try atomic_file.finish();
- return PrevStatus.stale;
+ return .stale;
}
pub const CopyFileError = File.OpenError || File.StatError ||
- AtomicFile.InitError || CopyFileRawError || AtomicFile.FinishError;
+ AtomicFile.InitError || AtomicFile.FinishError ||
+ File.ReadError || File.WriteError;
-/// Guaranteed to be atomic.
-/// On Linux, until https://patchwork.kernel.org/patch/9636735/ is merged and readily available,
-/// there is a possibility of power loss or application termination leaving temporary files present
-/// in the same directory as dest_path.
-/// On Windows, both paths should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/).
-/// On WASI, both paths should be encoded as valid UTF-8.
-/// On other platforms, both paths are an opaque sequence of bytes with no particular encoding.
+/// Atomically creates a new file at `dest_path` within `dest_dir` with the
+/// same contents as `source_path` within `source_dir`, overwriting any already
+/// existing file.
+///
+/// On Linux, until https://patchwork.kernel.org/patch/9636735/ is merged and
+/// readily available, there is a possibility of power loss or application
+/// termination leaving temporary files present in the same directory as
+/// dest_path.
+///
+/// On Windows, both paths should be encoded as
+/// [WTF-8](https://simonsapin.github.io/wtf-8/). On WASI, both paths should be
+/// encoded as valid UTF-8. On other platforms, both paths are an opaque
+/// sequence of bytes with no particular encoding.
pub fn copyFile(
source_dir: Dir,
source_path: []const u8,
@@ -2578,79 +2612,34 @@ pub fn copyFile(
dest_path: []const u8,
options: CopyFileOptions,
) CopyFileError!void {
- var in_file = try source_dir.openFile(source_path, .{});
- defer in_file.close();
+ var file_reader: File.Reader = .init(try source_dir.openFile(source_path, .{}), &.{});
+ defer file_reader.file.close();
- var size: ?u64 = null;
const mode = options.override_mode orelse blk: {
- const st = try in_file.stat();
- size = st.size;
+ const st = try file_reader.file.stat();
+ file_reader.size = st.size;
break :blk st.mode;
};
- var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode });
+ var buffer: [1024]u8 = undefined; // Used only when direct fd-to-fd is not available.
+ var atomic_file = try dest_dir.atomicFile(dest_path, .{
+ .mode = mode,
+ .write_buffer = &buffer,
+ });
defer atomic_file.deinit();
- try copy_file(in_file.handle, atomic_file.file.handle, size);
- try atomic_file.finish();
-}
-
-const CopyFileRawError = error{SystemResources} || posix.CopyFileRangeError || posix.SendFileError;
-
-// Transfer all the data between two file descriptors in the most efficient way.
-// The copy starts at offset 0, the initial offsets are preserved.
-// No metadata is transferred over.
-fn copy_file(fd_in: posix.fd_t, fd_out: posix.fd_t, maybe_size: ?u64) CopyFileRawError!void {
- if (builtin.target.os.tag.isDarwin()) {
- const rc = posix.system.fcopyfile(fd_in, fd_out, null, .{ .DATA = true });
- switch (posix.errno(rc)) {
- .SUCCESS => return,
- .INVAL => unreachable,
- .NOMEM => return error.SystemResources,
- // The source file is not a directory, symbolic link, or regular file.
- // Try with the fallback path before giving up.
- .OPNOTSUPP => {},
- else => |err| return posix.unexpectedErrno(err),
- }
- }
-
- if (native_os == .linux) {
- // Try copy_file_range first as that works at the FS level and is the
- // most efficient method (if available).
- var offset: u64 = 0;
- cfr_loop: while (true) {
- // The kernel checks the u64 value `offset+count` for overflow, use
- // a 32 bit value so that the syscall won't return EINVAL except for
- // impossibly large files (> 2^64-1 - 2^32-1).
- const amt = try posix.copy_file_range(fd_in, offset, fd_out, offset, std.math.maxInt(u32), 0);
- // Terminate as soon as we have copied size bytes or no bytes
- if (maybe_size) |s| {
- if (s == amt) break :cfr_loop;
- }
- if (amt == 0) break :cfr_loop;
- offset += amt;
- }
- return;
- }
+ _ = atomic_file.file_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) {
+ error.ReadFailed => return file_reader.err.?,
+ error.WriteFailed => return atomic_file.file_writer.err.?,
+ };
- // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the
- // fallback code will copy the contents chunk by chunk.
- const empty_iovec = [0]posix.iovec_const{};
- var offset: u64 = 0;
- sendfile_loop: while (true) {
- const amt = try posix.sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0);
- // Terminate as soon as we have copied size bytes or no bytes
- if (maybe_size) |s| {
- if (s == amt) break :sendfile_loop;
- }
- if (amt == 0) break :sendfile_loop;
- offset += amt;
- }
+ try atomic_file.finish();
}
pub const AtomicFileOptions = struct {
mode: File.Mode = File.default_mode,
make_path: bool = false,
+ write_buffer: []u8,
};
/// Directly access the `.file` field, and then call `AtomicFile.finish` to
@@ -2668,9 +2657,9 @@ pub fn atomicFile(self: Dir, dest_path: []const u8, options: AtomicFileOptions)
else
try self.openDir(dirname, .{});
- return AtomicFile.init(fs.path.basename(dest_path), options.mode, dir, true);
+ return .init(fs.path.basename(dest_path), options.mode, dir, true, options.write_buffer);
} else {
- return AtomicFile.init(dest_path, options.mode, self, false);
+ return .init(dest_path, options.mode, self, false, options.write_buffer);
}
}
@@ -2768,30 +2757,3 @@ pub fn setPermissions(self: Dir, permissions: Permissions) SetPermissionsError!v
const file: File = .{ .handle = self.fd };
try file.setPermissions(permissions);
}
-
-const Metadata = File.Metadata;
-pub const MetadataError = File.MetadataError;
-
-/// Returns a `Metadata` struct, representing the permissions on the directory
-pub fn metadata(self: Dir) MetadataError!Metadata {
- const file: File = .{ .handle = self.fd };
- return try file.metadata();
-}
-
-const Dir = @This();
-const builtin = @import("builtin");
-const std = @import("../std.zig");
-const File = std.fs.File;
-const AtomicFile = std.fs.AtomicFile;
-const base64_encoder = fs.base64_encoder;
-const crypto = std.crypto;
-const posix = std.posix;
-const mem = std.mem;
-const path = fs.path;
-const fs = std.fs;
-const Allocator = std.mem.Allocator;
-const assert = std.debug.assert;
-const linux = std.os.linux;
-const windows = std.os.windows;
-const native_os = builtin.os.tag;
-const have_flock = @TypeOf(posix.system.flock) != void;
diff --git a/lib/std/fs/File.zig b/lib/std/fs/File.zig
@@ -1089,113 +1089,6 @@ pub fn copyRangeAll(in: File, in_offset: u64, out: File, out_offset: u64, len: u
return total_bytes_copied;
}
-/// Deprecated in favor of `Writer`.
-pub const WriteFileOptions = struct {
- in_offset: u64 = 0,
- in_len: ?u64 = null,
- headers_and_trailers: []posix.iovec_const = &[0]posix.iovec_const{},
- header_count: usize = 0,
-};
-
-/// Deprecated in favor of `Writer`.
-pub const WriteFileError = ReadError || error{EndOfStream} || WriteError;
-
-/// Deprecated in favor of `Writer`.
-pub fn writeFileAll(self: File, in_file: File, args: WriteFileOptions) WriteFileError!void {
- return self.writeFileAllSendfile(in_file, args) catch |err| switch (err) {
- error.Unseekable,
- error.FastOpenAlreadyInProgress,
- error.MessageTooBig,
- error.FileDescriptorNotASocket,
- error.NetworkUnreachable,
- error.NetworkSubsystemFailed,
- error.ConnectionRefused,
- => return self.writeFileAllUnseekable(in_file, args),
- else => |e| return e,
- };
-}
-
-/// Deprecated in favor of `Writer`.
-pub fn writeFileAllUnseekable(self: File, in_file: File, args: WriteFileOptions) WriteFileError!void {
- const headers = args.headers_and_trailers[0..args.header_count];
- const trailers = args.headers_and_trailers[args.header_count..];
- try self.writevAll(headers);
- try in_file.deprecatedReader().skipBytes(args.in_offset, .{ .buf_size = 4096 });
- var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
- if (args.in_len) |len| {
- var stream = std.io.limitedReader(in_file.deprecatedReader(), len);
- try fifo.pump(stream.reader(), self.deprecatedWriter());
- } else {
- try fifo.pump(in_file.deprecatedReader(), self.deprecatedWriter());
- }
- try self.writevAll(trailers);
-}
-
-/// Deprecated in favor of `Writer`.
-fn writeFileAllSendfile(self: File, in_file: File, args: WriteFileOptions) posix.SendFileError!void {
- const count = blk: {
- if (args.in_len) |l| {
- if (l == 0) {
- return self.writevAll(args.headers_and_trailers);
- } else {
- break :blk l;
- }
- } else {
- break :blk 0;
- }
- };
- const headers = args.headers_and_trailers[0..args.header_count];
- const trailers = args.headers_and_trailers[args.header_count..];
- const zero_iovec = &[0]posix.iovec_const{};
- // When reading the whole file, we cannot put the trailers in the sendfile() syscall,
- // because we have no way to determine whether a partial write is past the end of the file or not.
- const trls = if (count == 0) zero_iovec else trailers;
- const offset = args.in_offset;
- const out_fd = self.handle;
- const in_fd = in_file.handle;
- const flags = 0;
- var amt: usize = 0;
- hdrs: {
- var i: usize = 0;
- while (i < headers.len) {
- amt = try posix.sendfile(out_fd, in_fd, offset, count, headers[i..], trls, flags);
- while (amt >= headers[i].len) {
- amt -= headers[i].len;
- i += 1;
- if (i >= headers.len) break :hdrs;
- }
- headers[i].base += amt;
- headers[i].len -= amt;
- }
- }
- if (count == 0) {
- var off: u64 = amt;
- while (true) {
- amt = try posix.sendfile(out_fd, in_fd, offset + off, 0, zero_iovec, zero_iovec, flags);
- if (amt == 0) break;
- off += amt;
- }
- } else {
- var off: u64 = amt;
- while (off < count) {
- amt = try posix.sendfile(out_fd, in_fd, offset + off, count - off, zero_iovec, trailers, flags);
- off += amt;
- }
- amt = @as(usize, @intCast(off - count));
- }
- var i: usize = 0;
- while (i < trailers.len) {
- while (amt >= trailers[i].len) {
- amt -= trailers[i].len;
- i += 1;
- if (i >= trailers.len) return;
- }
- trailers[i].base += amt;
- trailers[i].len -= amt;
- amt = try posix.writev(self.handle, trailers[i..]);
- }
-}
-
/// Deprecated in favor of `Reader`.
pub const DeprecatedReader = io.GenericReader(File, ReadError, read);
@@ -1242,7 +1135,7 @@ pub const Reader = struct {
err: ?ReadError = null,
mode: Reader.Mode = .positional,
/// Tracks the true seek position in the file. To obtain the logical
- /// position, subtract the buffer size from this value.
+ /// position, use `logicalPos`.
pos: u64 = 0,
size: ?u64 = null,
size_err: ?GetEndPosError = null,
@@ -1335,14 +1228,12 @@ pub const Reader = struct {
pub fn seekBy(r: *Reader, offset: i64) Reader.SeekError!void {
switch (r.mode) {
.positional, .positional_reading => {
- // TODO: make += operator allow any integer types
- r.pos = @intCast(@as(i64, @intCast(r.pos)) + offset);
+ setPosAdjustingBuffer(r, @intCast(@as(i64, @intCast(r.pos)) + offset));
},
.streaming, .streaming_reading => {
const seek_err = r.seek_err orelse e: {
if (posix.lseek_CUR(r.file.handle, offset)) |_| {
- // TODO: make += operator allow any integer types
- r.pos = @intCast(@as(i64, @intCast(r.pos)) + offset);
+ setPosAdjustingBuffer(r, @intCast(@as(i64, @intCast(r.pos)) + offset));
return;
} else |err| {
r.seek_err = err;
@@ -1358,6 +1249,8 @@ pub const Reader = struct {
r.pos += n;
remaining -= n;
}
+ r.interface.seek = 0;
+ r.interface.end = 0;
},
.failure => return r.seek_err.?,
}
@@ -1366,7 +1259,7 @@ pub const Reader = struct {
pub fn seekTo(r: *Reader, offset: u64) Reader.SeekError!void {
switch (r.mode) {
.positional, .positional_reading => {
- r.pos = offset;
+ setPosAdjustingBuffer(r, offset);
},
.streaming, .streaming_reading => {
if (offset >= r.pos) return Reader.seekBy(r, @intCast(offset - r.pos));
@@ -1375,12 +1268,28 @@ pub const Reader = struct {
r.seek_err = err;
return err;
};
- r.pos = offset;
+ setPosAdjustingBuffer(r, offset);
},
.failure => return r.seek_err.?,
}
}
+ pub fn logicalPos(r: *const Reader) u64 {
+ return r.pos - r.interface.bufferedLen();
+ }
+
+ fn setPosAdjustingBuffer(r: *Reader, offset: u64) void {
+ const logical_pos = logicalPos(r);
+ if (offset < logical_pos or offset >= r.pos) {
+ r.interface.seek = 0;
+ r.interface.end = 0;
+ r.pos = offset;
+ } else {
+ const logical_delta: usize = @intCast(offset - logical_pos);
+ r.interface.seek += logical_delta;
+ }
+ }
+
/// Number of slices to store on the stack, when trying to send as many byte
/// vectors through the underlying read calls as possible.
const max_buffers_len = 16;
@@ -1526,7 +1435,7 @@ pub const Reader = struct {
}
return 0;
};
- const n = @min(size - pos, std.math.maxInt(i64), @intFromEnum(limit));
+ const n = @min(size - pos, maxInt(i64), @intFromEnum(limit));
file.seekBy(n) catch |err| {
r.seek_err = err;
return 0;
@@ -1715,7 +1624,6 @@ pub const Writer = struct {
const pattern = data[data.len - 1];
if (pattern.len == 0 or splat == 0) return 0;
const n = windows.WriteFile(handle, pattern, null) catch |err| {
- std.debug.print("windows write file failed3: {t}\n", .{err});
w.err = err;
return error.WriteFailed;
};
@@ -1817,18 +1725,141 @@ pub const Writer = struct {
file_reader: *Reader,
limit: std.io.Limit,
) std.io.Writer.FileError!usize {
+ const reader_buffered = file_reader.interface.buffered();
+ if (reader_buffered.len >= @intFromEnum(limit))
+ return sendFileBuffered(io_w, file_reader, reader_buffered);
+ const writer_buffered = io_w.buffered();
+ const file_limit = @intFromEnum(limit) - reader_buffered.len;
const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w));
const out_fd = w.file.handle;
const in_fd = file_reader.file.handle;
- // TODO try using copy_file_range on FreeBSD
- // TODO try using sendfile on macOS
- // TODO try using sendfile on FreeBSD
+
+ if (file_reader.size) |size| {
+ if (size - file_reader.pos == 0) {
+ if (reader_buffered.len != 0) {
+ return sendFileBuffered(io_w, file_reader, reader_buffered);
+ } else {
+ return error.EndOfStream;
+ }
+ }
+ }
+
+ if (native_os == .freebsd and w.mode == .streaming) sf: {
+ // Try using sendfile on FreeBSD.
+ if (w.sendfile_err != null) break :sf;
+ const offset = std.math.cast(std.c.off_t, file_reader.pos) orelse break :sf;
+ var hdtr_data: std.c.sf_hdtr = undefined;
+ var headers: [2]posix.iovec_const = undefined;
+ var headers_i: u8 = 0;
+ if (writer_buffered.len != 0) {
+ headers[headers_i] = .{ .base = writer_buffered.ptr, .len = writer_buffered.len };
+ headers_i += 1;
+ }
+ if (reader_buffered.len != 0) {
+ headers[headers_i] = .{ .base = reader_buffered.ptr, .len = reader_buffered.len };
+ headers_i += 1;
+ }
+ const hdtr: ?*std.c.sf_hdtr = if (headers_i == 0) null else b: {
+ hdtr_data = .{
+ .headers = &headers,
+ .hdr_cnt = headers_i,
+ .trailers = null,
+ .trl_cnt = 0,
+ };
+ break :b &hdtr_data;
+ };
+ var sbytes: std.c.off_t = undefined;
+ const nbytes: usize = @min(file_limit, maxInt(usize));
+ const flags = 0;
+ switch (posix.errno(std.c.sendfile(in_fd, out_fd, offset, nbytes, hdtr, &sbytes, flags))) {
+ .SUCCESS, .INTR => {},
+ .INVAL, .OPNOTSUPP, .NOTSOCK, .NOSYS => w.sendfile_err = error.UnsupportedOperation,
+ .BADF => if (builtin.mode == .Debug) @panic("race condition") else {
+ w.sendfile_err = error.Unexpected;
+ },
+ .FAULT => if (builtin.mode == .Debug) @panic("segmentation fault") else {
+ w.sendfile_err = error.Unexpected;
+ },
+ .NOTCONN => w.sendfile_err = error.BrokenPipe,
+ .AGAIN, .BUSY => if (sbytes == 0) {
+ w.sendfile_err = error.WouldBlock;
+ },
+ .IO => w.sendfile_err = error.InputOutput,
+ .PIPE => w.sendfile_err = error.BrokenPipe,
+ .NOBUFS => w.sendfile_err = error.SystemResources,
+ else => |err| w.sendfile_err = posix.unexpectedErrno(err),
+ }
+ if (sbytes == 0) {
+ file_reader.size = file_reader.pos;
+ return error.EndOfStream;
+ }
+ const consumed = io_w.consume(@intCast(sbytes));
+ file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed;
+ return consumed;
+ }
+
+ if (native_os.isDarwin() and w.mode == .streaming) sf: {
+ // Try using sendfile on macOS.
+ if (w.sendfile_err != null) break :sf;
+ const offset = std.math.cast(std.c.off_t, file_reader.pos) orelse break :sf;
+ var hdtr_data: std.c.sf_hdtr = undefined;
+ var headers: [2]posix.iovec_const = undefined;
+ var headers_i: u8 = 0;
+ if (writer_buffered.len != 0) {
+ headers[headers_i] = .{ .base = writer_buffered.ptr, .len = writer_buffered.len };
+ headers_i += 1;
+ }
+ if (reader_buffered.len != 0) {
+ headers[headers_i] = .{ .base = reader_buffered.ptr, .len = reader_buffered.len };
+ headers_i += 1;
+ }
+ const hdtr: ?*std.c.sf_hdtr = if (headers_i == 0) null else b: {
+ hdtr_data = .{
+ .headers = &headers,
+ .hdr_cnt = headers_i,
+ .trailers = null,
+ .trl_cnt = 0,
+ };
+ break :b &hdtr_data;
+ };
+ const max_count = maxInt(i32); // Avoid EINVAL.
+ var len: std.c.off_t = @min(file_limit, max_count);
+ const flags = 0;
+ switch (posix.errno(std.c.sendfile(in_fd, out_fd, offset, &len, hdtr, flags))) {
+ .SUCCESS, .INTR => {},
+ .OPNOTSUPP, .NOTSOCK, .NOSYS => w.sendfile_err = error.UnsupportedOperation,
+ .BADF => if (builtin.mode == .Debug) @panic("race condition") else {
+ w.sendfile_err = error.Unexpected;
+ },
+ .FAULT => if (builtin.mode == .Debug) @panic("segmentation fault") else {
+ w.sendfile_err = error.Unexpected;
+ },
+ .INVAL => if (builtin.mode == .Debug) @panic("invalid API usage") else {
+ w.sendfile_err = error.Unexpected;
+ },
+ .NOTCONN => w.sendfile_err = error.BrokenPipe,
+ .AGAIN => if (len == 0) {
+ w.sendfile_err = error.WouldBlock;
+ },
+ .IO => w.sendfile_err = error.InputOutput,
+ .PIPE => w.sendfile_err = error.BrokenPipe,
+ else => |err| w.sendfile_err = posix.unexpectedErrno(err),
+ }
+ if (len == 0) {
+ file_reader.size = file_reader.pos;
+ return error.EndOfStream;
+ }
+ const consumed = io_w.consume(@bitCast(len));
+ file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed;
+ return consumed;
+ }
+
if (native_os == .linux and w.mode == .streaming) sf: {
// Try using sendfile on Linux.
if (w.sendfile_err != null) break :sf;
// Linux sendfile does not support headers.
- const buffered = limit.slice(file_reader.interface.buffer);
- if (io_w.end != 0 or buffered.len != 0) return drain(io_w, &.{buffered}, 1);
+ if (writer_buffered.len != 0 or reader_buffered.len != 0)
+ return sendFileBuffered(io_w, file_reader, reader_buffered);
const max_count = 0x7ffff000; // Avoid EINVAL.
var off: std.os.linux.off_t = undefined;
const off_ptr: ?*std.os.linux.off_t, const count: usize = switch (file_reader.mode) {
@@ -1875,6 +1906,7 @@ pub const Writer = struct {
w.pos += n;
return n;
}
+
const copy_file_range = switch (native_os) {
.freebsd => std.os.freebsd.copy_file_range,
.linux => if (std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 })) std.os.linux.wrapped.copy_file_range else {},
@@ -1882,8 +1914,8 @@ pub const Writer = struct {
};
if (@TypeOf(copy_file_range) != void) cfr: {
if (w.copy_file_range_err != null) break :cfr;
- const buffered = limit.slice(file_reader.interface.buffer);
- if (io_w.end != 0 or buffered.len != 0) return drain(io_w, &.{buffered}, 1);
+ if (writer_buffered.len != 0 or reader_buffered.len != 0)
+ return sendFileBuffered(io_w, file_reader, reader_buffered);
var off_in: i64 = undefined;
var off_out: i64 = undefined;
const off_in_ptr: ?*i64 = switch (file_reader.mode) {
@@ -1922,6 +1954,9 @@ pub const Writer = struct {
if (file_reader.pos != 0) break :fcf;
if (w.pos != 0) break :fcf;
if (limit != .unlimited) break :fcf;
+ const size = file_reader.getSize() catch break :fcf;
+ if (writer_buffered.len != 0 or reader_buffered.len != 0)
+ return sendFileBuffered(io_w, file_reader, reader_buffered);
const rc = std.c.fcopyfile(in_fd, out_fd, null, .{ .DATA = true });
switch (posix.errno(rc)) {
.SUCCESS => {},
@@ -1942,15 +1977,24 @@ pub const Writer = struct {
return 0;
},
}
- const n = if (file_reader.size) |size| size else @panic("TODO figure out how much copied");
- file_reader.pos = n;
- w.pos = n;
- return n;
+ file_reader.pos = size;
+ w.pos = size;
+ return size;
}
return error.Unimplemented;
}
+ fn sendFileBuffered(
+ io_w: *std.io.Writer,
+ file_reader: *Reader,
+ reader_buffered: []const u8,
+ ) std.io.Writer.FileError!usize {
+ const n = try drain(io_w, &.{reader_buffered}, 1);
+ file_reader.seekTo(file_reader.pos + n) catch return error.ReadFailed;
+ return n;
+ }
+
pub fn seekTo(w: *Writer, offset: u64) SeekError!void {
switch (w.mode) {
.positional, .positional_reading => {
diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig
@@ -1499,32 +1499,18 @@ test "sendfile" {
const header2 = "second header\n";
const trailer1 = "trailer1\n";
const trailer2 = "second trailer\n";
- var hdtr = [_]posix.iovec_const{
- .{
- .base = header1,
- .len = header1.len,
- },
- .{
- .base = header2,
- .len = header2.len,
- },
- .{
- .base = trailer1,
- .len = trailer1.len,
- },
- .{
- .base = trailer2,
- .len = trailer2.len,
- },
- };
+ var headers: [2][]const u8 = .{ header1, header2 };
+ var trailers: [2][]const u8 = .{ trailer1, trailer2 };
var written_buf: [100]u8 = undefined;
- try dest_file.writeFileAll(src_file, .{
- .in_offset = 1,
- .in_len = 10,
- .headers_and_trailers = &hdtr,
- .header_count = 2,
- });
+ var file_reader = src_file.reader(&.{});
+ var fallback_buffer: [50]u8 = undefined;
+ var file_writer = dest_file.writer(&fallback_buffer);
+ try file_writer.interface.writeVecAll(&headers);
+ try file_reader.seekTo(1);
+ try testing.expectEqual(10, try file_writer.interface.sendFileAll(&file_reader, .limited(10)));
+ try file_writer.interface.writeVecAll(&trailers);
+ try file_writer.interface.flush();
const amt = try dest_file.preadAll(&written_buf, 0);
try testing.expectEqualStrings("header1\nsecond header\nine1\nsecontrailer1\nsecond trailer\n", written_buf[0..amt]);
}
@@ -1595,9 +1581,10 @@ test "AtomicFile" {
;
{
- var af = try ctx.dir.atomicFile(test_out_file, .{});
+ var buffer: [100]u8 = undefined;
+ var af = try ctx.dir.atomicFile(test_out_file, .{ .write_buffer = &buffer });
defer af.deinit();
- try af.file.writeAll(test_content);
+ try af.file_writer.interface.writeAll(test_content);
try af.finish();
}
const content = try ctx.dir.readFileAlloc(allocator, test_out_file, 9999);
@@ -2073,7 +2060,7 @@ test "invalid UTF-8/WTF-8 paths" {
}
test "read file non vectored" {
- var tmp_dir = std.testing.tmpDir(.{});
+ var tmp_dir = testing.tmpDir(.{});
defer tmp_dir.cleanup();
const contents = "hello, world!\n";
@@ -2098,6 +2085,47 @@ test "read file non vectored" {
else => |e| return e,
};
}
- try std.testing.expectEqualStrings(contents, w.buffered());
- try std.testing.expectEqual(contents.len, i);
+ try testing.expectEqualStrings(contents, w.buffered());
+ try testing.expectEqual(contents.len, i);
+}
+
+test "seek keeping partial buffer" {
+ var tmp_dir = testing.tmpDir(.{});
+ defer tmp_dir.cleanup();
+
+ const contents = "0123456789";
+
+ const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });
+ defer file.close();
+ {
+ var file_writer: std.fs.File.Writer = .init(file, &.{});
+ try file_writer.interface.writeAll(contents);
+ try file_writer.interface.flush();
+ }
+
+ var read_buffer: [3]u8 = undefined;
+ var file_reader: std.fs.File.Reader = .init(file, &read_buffer);
+
+ try testing.expectEqual(0, file_reader.logicalPos());
+
+ var buf: [4]u8 = undefined;
+ try file_reader.interface.readSliceAll(&buf);
+
+ if (file_reader.interface.bufferedLen() != 3) {
+ // Pass the test if the OS doesn't give us vectored reads.
+ return;
+ }
+
+ try testing.expectEqual(4, file_reader.logicalPos());
+ try testing.expectEqual(7, file_reader.pos);
+ try file_reader.seekTo(6);
+ try testing.expectEqual(6, file_reader.logicalPos());
+ try testing.expectEqual(7, file_reader.pos);
+
+ try testing.expectEqualStrings("0123", &buf);
+
+ const n = try file_reader.interface.readSliceShort(&buf);
+ try testing.expectEqual(4, n);
+
+ try testing.expectEqualStrings("6789", &buf);
}
diff --git a/lib/std/json.zig b/lib/std/json.zig
@@ -69,7 +69,6 @@ pub const ArrayHashMap = @import("json/hashmap.zig").ArrayHashMap;
pub const Scanner = @import("json/Scanner.zig");
pub const validate = Scanner.validate;
pub const Error = Scanner.Error;
-pub const reader = Scanner.reader;
pub const default_buffer_size = Scanner.default_buffer_size;
pub const Token = Scanner.Token;
pub const TokenType = Scanner.TokenType;
diff --git a/lib/std/posix.zig b/lib/std/posix.zig
@@ -6326,295 +6326,6 @@ pub fn send(
};
}
-pub const SendFileError = PReadError || WriteError || SendError;
-
-/// Transfer data between file descriptors, with optional headers and trailers.
-///
-/// Returns the number of bytes written, which can be zero.
-///
-/// The `sendfile` call copies `in_len` bytes from one file descriptor to another. When possible,
-/// this is done within the operating system kernel, which can provide better performance
-/// characteristics than transferring data from kernel to user space and back, such as with
-/// `read` and `write` calls. When `in_len` is `0`, it means to copy until the end of the input file has been
-/// reached. Note, however, that partial writes are still possible in this case.
-///
-/// `in_fd` must be a file descriptor opened for reading, and `out_fd` must be a file descriptor
-/// opened for writing. They may be any kind of file descriptor; however, if `in_fd` is not a regular
-/// file system file, it may cause this function to fall back to calling `read` and `write`, in which case
-/// atomicity guarantees no longer apply.
-///
-/// Copying begins reading at `in_offset`. The input file descriptor seek position is ignored and not updated.
-/// If the output file descriptor has a seek position, it is updated as bytes are written. When
-/// `in_offset` is past the end of the input file, it successfully reads 0 bytes.
-///
-/// `flags` has different meanings per operating system; refer to the respective man pages.
-///
-/// These systems support atomically sending everything, including headers and trailers:
-/// * macOS
-/// * FreeBSD
-///
-/// These systems support in-kernel data copying, but headers and trailers are not sent atomically:
-/// * Linux
-///
-/// Other systems fall back to calling `read` / `write`.
-///
-/// Linux has a limit on how many bytes may be transferred in one `sendfile` call, which is `0x7ffff000`
-/// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as
-/// well as stuffing the errno codes into the last `4096` values. This is noted on the `sendfile` man page.
-/// The limit on Darwin is `0x7fffffff`, trying to write more than that returns EINVAL.
-/// The corresponding POSIX limit on this is `maxInt(isize)`.
-pub fn sendfile(
- out_fd: fd_t,
- in_fd: fd_t,
- in_offset: u64,
- in_len: u64,
- headers: []const iovec_const,
- trailers: []const iovec_const,
- flags: u32,
-) SendFileError!usize {
- var header_done = false;
- var total_written: usize = 0;
-
- // Prevents EOVERFLOW.
- const size_t = std.meta.Int(.unsigned, @typeInfo(usize).int.bits - 1);
- const max_count = switch (native_os) {
- .linux => 0x7ffff000,
- .macos, .ios, .watchos, .tvos, .visionos => maxInt(i32),
- else => maxInt(size_t),
- };
-
- switch (native_os) {
- .linux => sf: {
- if (headers.len != 0) {
- const amt = try writev(out_fd, headers);
- total_written += amt;
- if (amt < count_iovec_bytes(headers)) return total_written;
- header_done = true;
- }
-
- // Here we match BSD behavior, making a zero count value send as many bytes as possible.
- const adjusted_count = if (in_len == 0) max_count else @min(in_len, max_count);
-
- const sendfile_sym = if (lfs64_abi) system.sendfile64 else system.sendfile;
- while (true) {
- var offset: off_t = @bitCast(in_offset);
- const rc = sendfile_sym(out_fd, in_fd, &offset, adjusted_count);
- switch (errno(rc)) {
- .SUCCESS => {
- const amt: usize = @bitCast(rc);
- total_written += amt;
- if (in_len == 0 and amt == 0) {
- // We have detected EOF from `in_fd`.
- break;
- } else if (amt < in_len) {
- return total_written;
- } else {
- break;
- }
- },
-
- .BADF => unreachable, // Always a race condition.
- .FAULT => unreachable, // Segmentation fault.
- .OVERFLOW => unreachable, // We avoid passing too large of a `count`.
- .NOTCONN => return error.BrokenPipe, // `out_fd` is an unconnected socket
-
- .INVAL => {
- // EINVAL could be any of the following situations:
- // * Descriptor is not valid or locked
- // * an mmap(2)-like operation is not available for in_fd
- // * count is negative
- // * out_fd has the APPEND flag set
- // Because of the "mmap(2)-like operation" possibility, we fall back to doing read/write
- // manually.
- break :sf;
- },
- .AGAIN => return error.WouldBlock,
- .IO => return error.InputOutput,
- .PIPE => return error.BrokenPipe,
- .NOMEM => return error.SystemResources,
- .NXIO => return error.Unseekable,
- .SPIPE => return error.Unseekable,
- else => |err| {
- unexpectedErrno(err) catch {};
- break :sf;
- },
- }
- }
-
- if (trailers.len != 0) {
- total_written += try writev(out_fd, trailers);
- }
-
- return total_written;
- },
- .freebsd => sf: {
- var hdtr_data: std.c.sf_hdtr = undefined;
- var hdtr: ?*std.c.sf_hdtr = null;
- if (headers.len != 0 or trailers.len != 0) {
- // Here we carefully avoid `@intCast` by returning partial writes when
- // too many io vectors are provided.
- const hdr_cnt = cast(u31, headers.len) orelse maxInt(u31);
- if (headers.len > hdr_cnt) return writev(out_fd, headers);
-
- const trl_cnt = cast(u31, trailers.len) orelse maxInt(u31);
-
- hdtr_data = std.c.sf_hdtr{
- .headers = headers.ptr,
- .hdr_cnt = hdr_cnt,
- .trailers = trailers.ptr,
- .trl_cnt = trl_cnt,
- };
- hdtr = &hdtr_data;
- }
-
- while (true) {
- var sbytes: off_t = undefined;
- const err = errno(system.sendfile(in_fd, out_fd, @bitCast(in_offset), @min(in_len, max_count), hdtr, &sbytes, flags));
- const amt: usize = @bitCast(sbytes);
- switch (err) {
- .SUCCESS => return amt,
-
- .BADF => unreachable, // Always a race condition.
- .FAULT => unreachable, // Segmentation fault.
- .NOTCONN => return error.BrokenPipe, // `out_fd` is an unconnected socket
-
- .INVAL, .OPNOTSUPP, .NOTSOCK, .NOSYS => {
- // EINVAL could be any of the following situations:
- // * The fd argument is not a regular file.
- // * The s argument is not a SOCK.STREAM type socket.
- // * The offset argument is negative.
- // Because of some of these possibilities, we fall back to doing read/write
- // manually, the same as ENOSYS.
- break :sf;
- },
-
- .INTR => if (amt != 0) return amt else continue,
-
- .AGAIN => if (amt != 0) {
- return amt;
- } else {
- return error.WouldBlock;
- },
-
- .BUSY => if (amt != 0) {
- return amt;
- } else {
- return error.WouldBlock;
- },
-
- .IO => return error.InputOutput,
- .NOBUFS => return error.SystemResources,
- .PIPE => return error.BrokenPipe,
-
- else => {
- unexpectedErrno(err) catch {};
- if (amt != 0) {
- return amt;
- } else {
- break :sf;
- }
- },
- }
- }
- },
- .macos, .ios, .tvos, .watchos, .visionos => sf: {
- var hdtr_data: std.c.sf_hdtr = undefined;
- var hdtr: ?*std.c.sf_hdtr = null;
- if (headers.len != 0 or trailers.len != 0) {
- // Here we carefully avoid `@intCast` by returning partial writes when
- // too many io vectors are provided.
- const hdr_cnt = cast(u31, headers.len) orelse maxInt(u31);
- if (headers.len > hdr_cnt) return writev(out_fd, headers);
-
- const trl_cnt = cast(u31, trailers.len) orelse maxInt(u31);
-
- hdtr_data = std.c.sf_hdtr{
- .headers = headers.ptr,
- .hdr_cnt = hdr_cnt,
- .trailers = trailers.ptr,
- .trl_cnt = trl_cnt,
- };
- hdtr = &hdtr_data;
- }
-
- while (true) {
- var sbytes: off_t = @min(in_len, max_count);
- const err = errno(system.sendfile(in_fd, out_fd, @bitCast(in_offset), &sbytes, hdtr, flags));
- const amt: usize = @bitCast(sbytes);
- switch (err) {
- .SUCCESS => return amt,
-
- .BADF => unreachable, // Always a race condition.
- .FAULT => unreachable, // Segmentation fault.
- .INVAL => unreachable,
- .NOTCONN => return error.BrokenPipe, // `out_fd` is an unconnected socket
-
- .OPNOTSUPP, .NOTSOCK, .NOSYS => break :sf,
-
- .INTR => if (amt != 0) return amt else continue,
-
- .AGAIN => if (amt != 0) {
- return amt;
- } else {
- return error.WouldBlock;
- },
-
- .IO => return error.InputOutput,
- .PIPE => return error.BrokenPipe,
-
- else => {
- unexpectedErrno(err) catch {};
- if (amt != 0) {
- return amt;
- } else {
- break :sf;
- }
- },
- }
- }
- },
- else => {}, // fall back to read/write
- }
-
- if (headers.len != 0 and !header_done) {
- const amt = try writev(out_fd, headers);
- total_written += amt;
- if (amt < count_iovec_bytes(headers)) return total_written;
- }
-
- rw: {
- var buf: [8 * 4096]u8 = undefined;
- // Here we match BSD behavior, making a zero count value send as many bytes as possible.
- const adjusted_count = if (in_len == 0) buf.len else @min(buf.len, in_len);
- const amt_read = try pread(in_fd, buf[0..adjusted_count], in_offset);
- if (amt_read == 0) {
- if (in_len == 0) {
- // We have detected EOF from `in_fd`.
- break :rw;
- } else {
- return total_written;
- }
- }
- const amt_written = try write(out_fd, buf[0..amt_read]);
- total_written += amt_written;
- if (amt_written < in_len or in_len == 0) return total_written;
- }
-
- if (trailers.len != 0) {
- total_written += try writev(out_fd, trailers);
- }
-
- return total_written;
-}
-
-fn count_iovec_bytes(iovs: []const iovec_const) usize {
- var count: usize = 0;
- for (iovs) |iov| {
- count += iov.len;
- }
- return count;
-}
-
pub const CopyFileRangeError = error{
FileTooBig,
InputOutput,
diff --git a/src/Builtin.zig b/src/Builtin.zig
@@ -342,9 +342,9 @@ pub fn updateFileOnDisk(file: *File, comp: *Compilation) !void {
}
// `make_path` matters because the dir hasn't actually been created yet.
- var af = try root_dir.atomicFile(sub_path, .{ .make_path = true });
+ var af = try root_dir.atomicFile(sub_path, .{ .make_path = true, .write_buffer = &.{} });
defer af.deinit();
- try af.file.writeAll(file.source.?);
+ try af.file_writer.interface.writeAll(file.source.?);
af.finish() catch |err| switch (err) {
error.AccessDenied => switch (builtin.os.tag) {
.windows => {
diff --git a/src/Compilation.zig b/src/Compilation.zig
@@ -3382,7 +3382,7 @@ pub fn saveState(comp: *Compilation) !void {
const gpa = comp.gpa;
- var bufs = std.ArrayList(std.posix.iovec_const).init(gpa);
+ var bufs = std.ArrayList([]const u8).init(gpa);
defer bufs.deinit();
var pt_headers = std.ArrayList(Header.PerThread).init(gpa);
@@ -3421,50 +3421,50 @@ pub fn saveState(comp: *Compilation) !void {
try bufs.ensureTotalCapacityPrecise(14 + 8 * pt_headers.items.len);
addBuf(&bufs, mem.asBytes(&header));
- addBuf(&bufs, mem.sliceAsBytes(pt_headers.items));
-
- addBuf(&bufs, mem.sliceAsBytes(ip.src_hash_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.src_hash_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.nav_val_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.nav_val_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.nav_ty_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.nav_ty_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.interned_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.interned_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.zon_file_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.zon_file_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.embed_file_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.embed_file_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.namespace_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.namespace_deps.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.namespace_name_deps.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.namespace_name_deps.values()));
-
- addBuf(&bufs, mem.sliceAsBytes(ip.first_dependency.keys()));
- addBuf(&bufs, mem.sliceAsBytes(ip.first_dependency.values()));
- addBuf(&bufs, mem.sliceAsBytes(ip.dep_entries.items));
- addBuf(&bufs, mem.sliceAsBytes(ip.free_dep_entries.items));
+ addBuf(&bufs, @ptrCast(pt_headers.items));
+
+ addBuf(&bufs, @ptrCast(ip.src_hash_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.src_hash_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.nav_val_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.nav_val_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.nav_ty_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.nav_ty_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.interned_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.interned_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.zon_file_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.zon_file_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.embed_file_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.embed_file_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.namespace_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.namespace_deps.values()));
+ addBuf(&bufs, @ptrCast(ip.namespace_name_deps.keys()));
+ addBuf(&bufs, @ptrCast(ip.namespace_name_deps.values()));
+
+ addBuf(&bufs, @ptrCast(ip.first_dependency.keys()));
+ addBuf(&bufs, @ptrCast(ip.first_dependency.values()));
+ addBuf(&bufs, @ptrCast(ip.dep_entries.items));
+ addBuf(&bufs, @ptrCast(ip.free_dep_entries.items));
for (ip.locals, pt_headers.items) |*local, pt_header| {
if (pt_header.intern_pool.limbs_len > 0) {
- addBuf(&bufs, mem.sliceAsBytes(local.shared.limbs.view().items(.@"0")[0..pt_header.intern_pool.limbs_len]));
+ addBuf(&bufs, @ptrCast(local.shared.limbs.view().items(.@"0")[0..pt_header.intern_pool.limbs_len]));
}
if (pt_header.intern_pool.extra_len > 0) {
- addBuf(&bufs, mem.sliceAsBytes(local.shared.extra.view().items(.@"0")[0..pt_header.intern_pool.extra_len]));
+ addBuf(&bufs, @ptrCast(local.shared.extra.view().items(.@"0")[0..pt_header.intern_pool.extra_len]));
}
if (pt_header.intern_pool.items_len > 0) {
- addBuf(&bufs, mem.sliceAsBytes(local.shared.items.view().items(.data)[0..pt_header.intern_pool.items_len]));
- addBuf(&bufs, mem.sliceAsBytes(local.shared.items.view().items(.tag)[0..pt_header.intern_pool.items_len]));
+ addBuf(&bufs, @ptrCast(local.shared.items.view().items(.data)[0..pt_header.intern_pool.items_len]));
+ addBuf(&bufs, @ptrCast(local.shared.items.view().items(.tag)[0..pt_header.intern_pool.items_len]));
}
if (pt_header.intern_pool.string_bytes_len > 0) {
addBuf(&bufs, local.shared.strings.view().items(.@"0")[0..pt_header.intern_pool.string_bytes_len]);
}
if (pt_header.intern_pool.tracked_insts_len > 0) {
- addBuf(&bufs, mem.sliceAsBytes(local.shared.tracked_insts.view().items(.@"0")[0..pt_header.intern_pool.tracked_insts_len]));
+ addBuf(&bufs, @ptrCast(local.shared.tracked_insts.view().items(.@"0")[0..pt_header.intern_pool.tracked_insts_len]));
}
if (pt_header.intern_pool.files_len > 0) {
- addBuf(&bufs, mem.sliceAsBytes(local.shared.files.view().items(.bin_digest)[0..pt_header.intern_pool.files_len]));
- addBuf(&bufs, mem.sliceAsBytes(local.shared.files.view().items(.root_type)[0..pt_header.intern_pool.files_len]));
+ addBuf(&bufs, @ptrCast(local.shared.files.view().items(.bin_digest)[0..pt_header.intern_pool.files_len]));
+ addBuf(&bufs, @ptrCast(local.shared.files.view().items(.root_type)[0..pt_header.intern_pool.files_len]));
}
}
@@ -3482,95 +3482,95 @@ pub fn saveState(comp: *Compilation) !void {
try bufs.ensureUnusedCapacity(85);
addBuf(&bufs, wasm.string_bytes.items);
// TODO make it well-defined memory layout
- //addBuf(&bufs, mem.sliceAsBytes(wasm.objects.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.func_types.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_function_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_function_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_functions.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_global_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_global_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_globals.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_table_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_table_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_tables.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_memory_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_memory_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_memories.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_relocations.items(.tag)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_relocations.items(.offset)));
+ //addBuf(&bufs, @ptrCast(wasm.objects.items));
+ addBuf(&bufs, @ptrCast(wasm.func_types.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_function_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_function_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.object_functions.items));
+ addBuf(&bufs, @ptrCast(wasm.object_global_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_global_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.object_globals.items));
+ addBuf(&bufs, @ptrCast(wasm.object_table_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_table_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.object_tables.items));
+ addBuf(&bufs, @ptrCast(wasm.object_memory_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_memory_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.object_memories.items));
+ addBuf(&bufs, @ptrCast(wasm.object_relocations.items(.tag)));
+ addBuf(&bufs, @ptrCast(wasm.object_relocations.items(.offset)));
// TODO handle the union safety field
- //addBuf(&bufs, mem.sliceAsBytes(wasm.object_relocations.items(.pointee)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_relocations.items(.addend)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_init_funcs.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_data_segments.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_datas.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_data_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_data_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_custom_segments.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_custom_segments.values()));
+ //addBuf(&bufs, @ptrCast(wasm.object_relocations.items(.pointee)));
+ addBuf(&bufs, @ptrCast(wasm.object_relocations.items(.addend)));
+ addBuf(&bufs, @ptrCast(wasm.object_init_funcs.items));
+ addBuf(&bufs, @ptrCast(wasm.object_data_segments.items));
+ addBuf(&bufs, @ptrCast(wasm.object_datas.items));
+ addBuf(&bufs, @ptrCast(wasm.object_data_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_data_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.object_custom_segments.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_custom_segments.values()));
// TODO make it well-defined memory layout
- // addBuf(&bufs, mem.sliceAsBytes(wasm.object_comdats.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_relocations_table.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_relocations_table.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_comdat_symbols.items(.kind)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_comdat_symbols.items(.index)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.out_relocs.items(.tag)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.out_relocs.items(.offset)));
+ // addBuf(&bufs, @ptrCast(wasm.object_comdats.items));
+ addBuf(&bufs, @ptrCast(wasm.object_relocations_table.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_relocations_table.values()));
+ addBuf(&bufs, @ptrCast(wasm.object_comdat_symbols.items(.kind)));
+ addBuf(&bufs, @ptrCast(wasm.object_comdat_symbols.items(.index)));
+ addBuf(&bufs, @ptrCast(wasm.out_relocs.items(.tag)));
+ addBuf(&bufs, @ptrCast(wasm.out_relocs.items(.offset)));
// TODO handle the union safety field
- //addBuf(&bufs, mem.sliceAsBytes(wasm.out_relocs.items(.pointee)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.out_relocs.items(.addend)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uav_fixups.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.nav_fixups.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.func_table_fixups.items));
+ //addBuf(&bufs, @ptrCast(wasm.out_relocs.items(.pointee)));
+ addBuf(&bufs, @ptrCast(wasm.out_relocs.items(.addend)));
+ addBuf(&bufs, @ptrCast(wasm.uav_fixups.items));
+ addBuf(&bufs, @ptrCast(wasm.nav_fixups.items));
+ addBuf(&bufs, @ptrCast(wasm.func_table_fixups.items));
if (is_obj) {
- addBuf(&bufs, mem.sliceAsBytes(wasm.navs_obj.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.navs_obj.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uavs_obj.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uavs_obj.values()));
+ addBuf(&bufs, @ptrCast(wasm.navs_obj.keys()));
+ addBuf(&bufs, @ptrCast(wasm.navs_obj.values()));
+ addBuf(&bufs, @ptrCast(wasm.uavs_obj.keys()));
+ addBuf(&bufs, @ptrCast(wasm.uavs_obj.values()));
} else {
- addBuf(&bufs, mem.sliceAsBytes(wasm.navs_exe.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.navs_exe.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uavs_exe.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uavs_exe.values()));
+ addBuf(&bufs, @ptrCast(wasm.navs_exe.keys()));
+ addBuf(&bufs, @ptrCast(wasm.navs_exe.values()));
+ addBuf(&bufs, @ptrCast(wasm.uavs_exe.keys()));
+ addBuf(&bufs, @ptrCast(wasm.uavs_exe.values()));
}
- addBuf(&bufs, mem.sliceAsBytes(wasm.overaligned_uavs.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.overaligned_uavs.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.zcu_funcs.keys()));
+ addBuf(&bufs, @ptrCast(wasm.overaligned_uavs.keys()));
+ addBuf(&bufs, @ptrCast(wasm.overaligned_uavs.values()));
+ addBuf(&bufs, @ptrCast(wasm.zcu_funcs.keys()));
// TODO handle the union safety field
- // addBuf(&bufs, mem.sliceAsBytes(wasm.zcu_funcs.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.nav_exports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.nav_exports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uav_exports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.uav_exports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.missing_exports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.function_exports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.function_exports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.hidden_function_exports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.hidden_function_exports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.global_exports.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.functions.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.function_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.function_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.data_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.data_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.data_segments.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.globals.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.global_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.global_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.tables.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.table_imports.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.table_imports.values()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.zcu_indirect_function_set.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_indirect_function_import_set.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.object_indirect_function_set.keys()));
- addBuf(&bufs, mem.sliceAsBytes(wasm.mir_instructions.items(.tag)));
+ // addBuf(&bufs, @ptrCast(wasm.zcu_funcs.values()));
+ addBuf(&bufs, @ptrCast(wasm.nav_exports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.nav_exports.values()));
+ addBuf(&bufs, @ptrCast(wasm.uav_exports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.uav_exports.values()));
+ addBuf(&bufs, @ptrCast(wasm.imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.missing_exports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.function_exports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.function_exports.values()));
+ addBuf(&bufs, @ptrCast(wasm.hidden_function_exports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.hidden_function_exports.values()));
+ addBuf(&bufs, @ptrCast(wasm.global_exports.items));
+ addBuf(&bufs, @ptrCast(wasm.functions.keys()));
+ addBuf(&bufs, @ptrCast(wasm.function_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.function_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.data_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.data_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.data_segments.keys()));
+ addBuf(&bufs, @ptrCast(wasm.globals.keys()));
+ addBuf(&bufs, @ptrCast(wasm.global_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.global_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.tables.keys()));
+ addBuf(&bufs, @ptrCast(wasm.table_imports.keys()));
+ addBuf(&bufs, @ptrCast(wasm.table_imports.values()));
+ addBuf(&bufs, @ptrCast(wasm.zcu_indirect_function_set.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_indirect_function_import_set.keys()));
+ addBuf(&bufs, @ptrCast(wasm.object_indirect_function_set.keys()));
+ addBuf(&bufs, @ptrCast(wasm.mir_instructions.items(.tag)));
// TODO handle the union safety field
- //addBuf(&bufs, mem.sliceAsBytes(wasm.mir_instructions.items(.data)));
- addBuf(&bufs, mem.sliceAsBytes(wasm.mir_extra.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.mir_locals.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.tag_name_bytes.items));
- addBuf(&bufs, mem.sliceAsBytes(wasm.tag_name_offs.items));
+ //addBuf(&bufs, @ptrCast(wasm.mir_instructions.items(.data)));
+ addBuf(&bufs, @ptrCast(wasm.mir_extra.items));
+ addBuf(&bufs, @ptrCast(wasm.mir_locals.items));
+ addBuf(&bufs, @ptrCast(wasm.tag_name_bytes.items));
+ addBuf(&bufs, @ptrCast(wasm.tag_name_offs.items));
// TODO add as header fields
// entry_resolution: FunctionImport.Resolution
@@ -3596,16 +3596,16 @@ pub fn saveState(comp: *Compilation) !void {
// Using an atomic file prevents a crash or power failure from corrupting
// the previous incremental compilation state.
- var af = try lf.emit.root_dir.handle.atomicFile(basename, .{});
+ var write_buffer: [1024]u8 = undefined;
+ var af = try lf.emit.root_dir.handle.atomicFile(basename, .{ .write_buffer = &write_buffer });
defer af.deinit();
- try af.file.pwritevAll(bufs.items, 0);
+ try af.file_writer.interface.writeVecAll(bufs.items);
try af.finish();
}
-fn addBuf(list: *std.ArrayList(std.posix.iovec_const), buf: []const u8) void {
- // Even when len=0, the undefined pointer might cause EFAULT.
+fn addBuf(list: *std.ArrayList([]const u8), buf: []const u8) void {
if (buf.len == 0) return;
- list.appendAssumeCapacity(.{ .base = buf.ptr, .len = buf.len });
+ list.appendAssumeCapacity(buf);
}
/// This function is temporally single-threaded.
diff --git a/src/fmt.zig b/src/fmt.zig
@@ -348,10 +348,10 @@ fn fmtPathFile(
try fmt.stdout_writer.interface.print("{s}\n", .{file_path});
fmt.any_error = true;
} else {
- var af = try dir.atomicFile(sub_path, .{ .mode = stat.mode });
+ var af = try dir.atomicFile(sub_path, .{ .mode = stat.mode, .write_buffer = &.{} });
defer af.deinit();
- try af.file.writeAll(fmt.out_buffer.getWritten());
+ try af.file_writer.interface.writeAll(fmt.out_buffer.getWritten());
try af.finish();
try fmt.stdout_writer.interface.print("{s}\n", .{file_path});
}
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
@@ -612,7 +612,6 @@ pub fn flush(
};
const emit = self.base.emit;
invalidateKernelCache(emit.root_dir.handle, emit.sub_path) catch |err| switch (err) {
- error.OutOfMemory => return error.OutOfMemory,
else => |e| return diags.fail("failed to invalidate kernel cache: {s}", .{@errorName(e)}),
};
}
diff --git a/src/main.zig b/src/main.zig
@@ -4624,7 +4624,9 @@ fn cmdTranslateC(
fatal("unable to open cached translated zig file '{s}{s}{s}': {s}", .{ path, fs.path.sep_str, out_zig_path, @errorName(err) });
};
defer zig_file.close();
- try fs.File.stdout().writeFileAll(zig_file, .{});
+ var stdout_writer = fs.File.stdout().writer(&stdout_buffer);
+ var file_reader = zig_file.reader(&.{});
+ _ = try stdout_writer.interface.sendFileAll(&file_reader, .unlimited);
return cleanExit();
}
}
diff --git a/test/incremental/fix_many_errors b/test/incremental/fix_many_errors
@@ -1,71 +0,0 @@
-#target=x86_64-linux-selfhosted
-#target=x86_64-linux-cbe
-#target=x86_64-windows-cbe
-#update=initial version
-#file=main.zig
-pub fn main() !void {}
-comptime { @compileError("c0"); }
-comptime { @compileError("c1"); }
-comptime { @compileError("c2"); }
-comptime { @compileError("c3"); }
-comptime { @compileError("c4"); }
-comptime { @compileError("c5"); }
-comptime { @compileError("c6"); }
-comptime { @compileError("c7"); }
-comptime { @compileError("c8"); }
-comptime { @compileError("c9"); }
-export fn f0() void { @compileError("f0"); }
-export fn f1() void { @compileError("f1"); }
-export fn f2() void { @compileError("f2"); }
-export fn f3() void { @compileError("f3"); }
-export fn f4() void { @compileError("f4"); }
-export fn f5() void { @compileError("f5"); }
-export fn f6() void { @compileError("f6"); }
-export fn f7() void { @compileError("f7"); }
-export fn f8() void { @compileError("f8"); }
-export fn f9() void { @compileError("f9"); }
-#expect_error=main.zig:2:12: error: c0
-#expect_error=main.zig:3:12: error: c1
-#expect_error=main.zig:4:12: error: c2
-#expect_error=main.zig:5:12: error: c3
-#expect_error=main.zig:6:12: error: c4
-#expect_error=main.zig:7:12: error: c5
-#expect_error=main.zig:8:12: error: c6
-#expect_error=main.zig:9:12: error: c7
-#expect_error=main.zig:10:12: error: c8
-#expect_error=main.zig:11:12: error: c9
-#expect_error=main.zig:12:23: error: f0
-#expect_error=main.zig:13:23: error: f1
-#expect_error=main.zig:14:23: error: f2
-#expect_error=main.zig:15:23: error: f3
-#expect_error=main.zig:16:23: error: f4
-#expect_error=main.zig:17:23: error: f5
-#expect_error=main.zig:18:23: error: f6
-#expect_error=main.zig:19:23: error: f7
-#expect_error=main.zig:20:23: error: f8
-#expect_error=main.zig:21:23: error: f9
-#update=fix all the errors
-#file=main.zig
-pub fn main() !void {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-comptime {}
-export fn f0() void {}
-export fn f1() void {}
-export fn f2() void {}
-export fn f3() void {}
-export fn f4() void {}
-export fn f5() void {}
-export fn f6() void {}
-export fn f7() void {}
-export fn f8() void {}
-export fn f9() void {}
-const std = @import("std");
-#expect_stdout=""
diff --git a/test/standalone/stack_iterator/build.zig b/test/standalone/stack_iterator/build.zig
@@ -65,69 +65,70 @@ pub fn build(b: *std.Build) void {
test_step.dependOn(&run_cmd.step);
}
- // Unwinding through a C shared library without a frame pointer (libc)
- //
- // getcontext version: libc
- //
- // Unwind info type:
- // - ELF: DWARF .eh_frame + .debug_frame
- // - MachO: __unwind_info encodings:
- // - x86_64: STACK_IMMD, STACK_IND
- // - aarch64: FRAMELESS, DWARF
- {
- const c_shared_lib = b.addLibrary(.{
- .linkage = .dynamic,
- .name = "c_shared_lib",
- .root_module = b.createModule(.{
- .root_source_file = null,
- .target = target,
- .optimize = optimize,
- .link_libc = true,
- .strip = false,
- }),
- });
-
- if (target.result.os.tag == .windows)
- c_shared_lib.root_module.addCMacro("LIB_API", "__declspec(dllexport)");
-
- c_shared_lib.root_module.addCSourceFile(.{
- .file = b.path("shared_lib.c"),
- .flags = &.{"-fomit-frame-pointer"},
- });
-
- const exe = b.addExecutable(.{
- .name = "shared_lib_unwind",
- .root_module = b.createModule(.{
- .root_source_file = b.path("shared_lib_unwind.zig"),
- .target = target,
- .optimize = optimize,
- .unwind_tables = if (target.result.os.tag.isDarwin()) .async else null,
- .omit_frame_pointer = true,
- }),
- // zig objcopy doesn't support incremental binaries
- .use_llvm = true,
- });
-
- exe.linkLibrary(c_shared_lib);
-
- const run_cmd = b.addRunArtifact(exe);
- test_step.dependOn(&run_cmd.step);
-
- // Separate debug info ELF file
- if (target.result.ofmt == .elf) {
- const filename = b.fmt("{s}_stripped", .{exe.out_filename});
- const stripped_exe = b.addObjCopy(exe.getEmittedBin(), .{
- .basename = filename, // set the name for the debuglink
- .compress_debug = true,
- .strip = .debug,
- .extract_to_separate_file = true,
- });
-
- const run_stripped = std.Build.Step.Run.create(b, b.fmt("run {s}", .{filename}));
- run_stripped.addFileArg(stripped_exe.getOutput());
- test_step.dependOn(&run_stripped.step);
- }
- }
+ // https://github.com/ziglang/zig/issues/24522
+ //// Unwinding through a C shared library without a frame pointer (libc)
+ ////
+ //// getcontext version: libc
+ ////
+ //// Unwind info type:
+ //// - ELF: DWARF .eh_frame + .debug_frame
+ //// - MachO: __unwind_info encodings:
+ //// - x86_64: STACK_IMMD, STACK_IND
+ //// - aarch64: FRAMELESS, DWARF
+ //{
+ // const c_shared_lib = b.addLibrary(.{
+ // .linkage = .dynamic,
+ // .name = "c_shared_lib",
+ // .root_module = b.createModule(.{
+ // .root_source_file = null,
+ // .target = target,
+ // .optimize = optimize,
+ // .link_libc = true,
+ // .strip = false,
+ // }),
+ // });
+
+ // if (target.result.os.tag == .windows)
+ // c_shared_lib.root_module.addCMacro("LIB_API", "__declspec(dllexport)");
+
+ // c_shared_lib.root_module.addCSourceFile(.{
+ // .file = b.path("shared_lib.c"),
+ // .flags = &.{"-fomit-frame-pointer"},
+ // });
+
+ // const exe = b.addExecutable(.{
+ // .name = "shared_lib_unwind",
+ // .root_module = b.createModule(.{
+ // .root_source_file = b.path("shared_lib_unwind.zig"),
+ // .target = target,
+ // .optimize = optimize,
+ // .unwind_tables = if (target.result.os.tag.isDarwin()) .async else null,
+ // .omit_frame_pointer = true,
+ // }),
+ // // zig objcopy doesn't support incremental binaries
+ // .use_llvm = true,
+ // });
+
+ // exe.linkLibrary(c_shared_lib);
+
+ // const run_cmd = b.addRunArtifact(exe);
+ // test_step.dependOn(&run_cmd.step);
+
+ // // Separate debug info ELF file
+ // if (target.result.ofmt == .elf) {
+ // const filename = b.fmt("{s}_stripped", .{exe.out_filename});
+ // const stripped_exe = b.addObjCopy(exe.getEmittedBin(), .{
+ // .basename = filename, // set the name for the debuglink
+ // .compress_debug = true,
+ // .strip = .debug,
+ // .extract_to_separate_file = true,
+ // });
+
+ // const run_stripped = std.Build.Step.Run.create(b, b.fmt("run {s}", .{filename}));
+ // run_stripped.addFileArg(stripped_exe.getOutput());
+ // test_step.dependOn(&run_stripped.step);
+ // }
+ //}
// Unwinding without libc/posix
//
diff --git a/tools/gen_stubs.zig b/tools/gen_stubs.zig
@@ -310,7 +310,8 @@ pub fn main() !void {
build_all_path, libc_so_path, @errorName(err),
});
};
- const header = try elf.Header.parse(elf_bytes[0..@sizeOf(elf.Elf64_Ehdr)]);
+ var stream: std.Io.Reader = .fixed(elf_bytes);
+ const header = try elf.Header.read(&stream);
const parse: Parse = .{
.arena = arena,
