From ffd53a459e1b665e5070987f68dd583171a12459 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 6 Mar 2024 21:15:36 -0700 Subject: [PATCH] -femit-docs: creating sources.tar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's always a good day when you get to use File.writeFileAll 😎 --- lib/compiler/std-docs.zig | 77 +------------------------------ lib/std/tar.zig | 36 ++++++++------- lib/std/tar/output.zig | 85 ++++++++++++++++++++++++++++++++++ src/Compilation.zig | 97 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 196 insertions(+), 99 deletions(-) create mode 100644 lib/std/tar/output.zig diff --git a/lib/compiler/std-docs.zig b/lib/compiler/std-docs.zig index 532c698c84..93a04a28e5 100644 --- a/lib/compiler/std-docs.zig +++ b/lib/compiler/std-docs.zig @@ -167,9 +167,8 @@ fn serveSourcesTar(request: *std.http.Server.Request, context: *Context) !void { const remainder = stat.size % 512; break :p if (remainder > 0) 512 - remainder else 0; }; - comptime assert(@sizeOf(TarHeader) == 512); - var file_header = TarHeader.init(); + var file_header = std.tar.output.Header.init(); file_header.typeflag = .regular; try file_header.setPath("std", entry.path); try file_header.setSize(stat.size); @@ -383,77 +382,3 @@ fn openBrowserTabThread(gpa: Allocator, url: []const u8) !void { try child.spawn(); _ = try child.wait(); } - -/// Forked from https://github.com/mattnite/tar/blob/main/src/main.zig which is -/// MIT licensed. -pub const TarHeader = extern struct { - name: [100]u8, - mode: [7:0]u8, - uid: [7:0]u8, - gid: [7:0]u8, - size: [11:0]u8, - mtime: [11:0]u8, - checksum: [7:0]u8, - typeflag: FileType, - linkname: [100]u8, - magic: [5:0]u8, - version: [2]u8, - uname: [31:0]u8, - gname: [31:0]u8, - devmajor: [7:0]u8, - devminor: [7:0]u8, - prefix: [155]u8, - pad: [12]u8, - - const FileType = enum(u8) { - regular = '0', - hard_link = '1', - symbolic_link = '2', - character = '3', - block = '4', - directory = '5', - fifo = '6', - reserved = '7', - pax_global = 'g', - extended = 'x', - _, - }; - - fn init() TarHeader { - var ret = std.mem.zeroes(TarHeader); - ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' }; - ret.version = [_:0]u8{ '0', '0' }; - return ret; - } - - fn setPath(self: *TarHeader, prefix: []const u8, path: []const u8) !void { - if (prefix.len + 1 + path.len > 100) { - var i: usize = 0; - while (i < path.len and path.len - i > 100) { - while (path[i] != '/') : (i += 1) {} - } - - _ = try std.fmt.bufPrint(&self.prefix, "{s}/{s}", .{ prefix, path[0..i] }); - _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]}); - } else { - _ = try std.fmt.bufPrint(&self.name, "{s}/{s}", .{ prefix, path }); - } - } - - fn setSize(self: *TarHeader, size: u64) !void { - _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size}); - } - - fn updateChecksum(self: *TarHeader) !void { - const offset = @offsetOf(TarHeader, "checksum"); - var checksum: usize = 0; - for (std.mem.asBytes(self), 0..) |val, i| { - checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum))) - ' ' - else - val; - } - - _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum}); - } -}; diff --git a/lib/std/tar.zig b/lib/std/tar.zig index af900b3880..121e7db248 100644 --- a/lib/std/tar.zig +++ b/lib/std/tar.zig @@ -1,23 +1,25 @@ -/// Tar archive is single ordinary file which can contain many files (or -/// directories, symlinks, ...). It's build by series of blocks each size of 512 -/// bytes. First block of each entry is header which defines type, name, size -/// permissions and other attributes. Header is followed by series of blocks of -/// file content, if any that entry has content. Content is padded to the block -/// size, so next header always starts at block boundary. -/// -/// This simple format is extended by GNU and POSIX pax extensions to support -/// file names longer than 256 bytes and additional attributes. -/// -/// This is not comprehensive tar parser. Here we are only file types needed to -/// support Zig package manager; normal file, directory, symbolic link. And -/// subset of attributes: name, size, permissions. -/// -/// GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html -/// pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13 -/// +//! Tar archive is single ordinary file which can contain many files (or +//! directories, symlinks, ...). It's build by series of blocks each size of 512 +//! bytes. First block of each entry is header which defines type, name, size +//! permissions and other attributes. Header is followed by series of blocks of +//! file content, if any that entry has content. Content is padded to the block +//! size, so next header always starts at block boundary. +//! +//! This simple format is extended by GNU and POSIX pax extensions to support +//! file names longer than 256 bytes and additional attributes. +//! +//! This is not comprehensive tar parser. Here we are only file types needed to +//! support Zig package manager; normal file, directory, symbolic link. And +//! subset of attributes: name, size, permissions. +//! +//! GNU tar reference: https://www.gnu.org/software/tar/manual/html_node/Standard.html +//! pax reference: https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13 + const std = @import("std.zig"); const assert = std.debug.assert; +pub const output = @import("tar/output.zig"); + pub const Options = struct { /// Number of directory levels to skip when extracting files. strip_components: u32 = 0, diff --git a/lib/std/tar/output.zig b/lib/std/tar/output.zig new file mode 100644 index 0000000000..73cfca58b1 --- /dev/null +++ b/lib/std/tar/output.zig @@ -0,0 +1,85 @@ +/// A struct that is exactly 512 bytes and matches tar file format. This is +/// intended to be used for outputting tar files; for parsing there is +/// `std.tar.Header`. +pub const Header = extern struct { + // This struct was originally copied from + // https://github.com/mattnite/tar/blob/main/src/main.zig which is MIT + // licensed. + + name: [100]u8, + mode: [7:0]u8, + uid: [7:0]u8, + gid: [7:0]u8, + size: [11:0]u8, + mtime: [11:0]u8, + checksum: [7:0]u8, + typeflag: FileType, + linkname: [100]u8, + magic: [5:0]u8, + version: [2]u8, + uname: [31:0]u8, + gname: [31:0]u8, + devmajor: [7:0]u8, + devminor: [7:0]u8, + prefix: [155]u8, + pad: [12]u8, + + pub const FileType = enum(u8) { + regular = '0', + hard_link = '1', + symbolic_link = '2', + character = '3', + block = '4', + directory = '5', + fifo = '6', + reserved = '7', + pax_global = 'g', + extended = 'x', + _, + }; + + pub fn init() Header { + var ret = std.mem.zeroes(Header); + ret.magic = [_:0]u8{ 'u', 's', 't', 'a', 'r' }; + ret.version = [_:0]u8{ '0', '0' }; + return ret; + } + + pub fn setPath(self: *Header, prefix: []const u8, path: []const u8) !void { + if (prefix.len + 1 + path.len > 100) { + var i: usize = 0; + while (i < path.len and path.len - i > 100) { + while (path[i] != '/') : (i += 1) {} + } + + _ = try std.fmt.bufPrint(&self.prefix, "{s}/{s}", .{ prefix, path[0..i] }); + _ = try std.fmt.bufPrint(&self.name, "{s}", .{path[i + 1 ..]}); + } else { + _ = try std.fmt.bufPrint(&self.name, "{s}/{s}", .{ prefix, path }); + } + } + + pub fn setSize(self: *Header, size: u64) !void { + _ = try std.fmt.bufPrint(&self.size, "{o:0>11}", .{size}); + } + + pub fn updateChecksum(self: *Header) !void { + const offset = @offsetOf(Header, "checksum"); + var checksum: usize = 0; + for (std.mem.asBytes(self), 0..) |val, i| { + checksum += if (i >= offset and i < offset + @sizeOf(@TypeOf(self.checksum))) + ' ' + else + val; + } + + _ = try std.fmt.bufPrint(&self.checksum, "{o:0>7}", .{checksum}); + } + + comptime { + assert(@sizeOf(Header) == 512); + } +}; + +const std = @import("../std.zig"); +const assert = std.debug.assert; diff --git a/src/Compilation.zig b/src/Compilation.zig index 46e5d7cde9..7e006539b6 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -733,6 +733,7 @@ pub const MiscTask = enum { compiler_rt, zig_libc, analyze_mod, + docs_copy, @"musl crti.o", @"musl crtn.o", @@ -3765,23 +3766,107 @@ fn taskDocsWasm(comp: *Compilation, wg: *WaitGroup) !void { fn workerDocsCopy(comp: *Compilation, wg: *WaitGroup) void { defer wg.finish(); + docsCopyFallible(comp) catch |err| { + return comp.lockAndSetMiscFailure( + .docs_copy, + "unable to copy autodocs artifacts: {s}", + .{@errorName(err)}, + ); + }; +} +fn docsCopyFallible(comp: *Compilation) anyerror!void { const emit = comp.docs_emit.?; var out_dir = emit.directory.handle.makeOpenPath(emit.sub_path, .{}) catch |err| { - // TODO create an error to be reported instead of logging - log.err("unable to create output directory '{}{s}': {s}", .{ - emit.directory, emit.sub_path, @errorName(err), - }); - return; + return comp.lockAndSetMiscFailure( + .docs_copy, + "unable to create output directory '{}{s}': {s}", + .{ emit.directory, emit.sub_path, @errorName(err) }, + ); }; defer out_dir.close(); for (&[_][]const u8{ "docs/main.js", "docs/index.html" }) |sub_path| { const basename = std.fs.path.basename(sub_path); comp.zig_lib_directory.handle.copyFile(sub_path, out_dir, basename, .{}) catch |err| { - log.err("unable to copy {s}: {s}", .{ sub_path, @errorName(err) }); + comp.lockAndSetMiscFailure(.docs_copy, "unable to copy {s}: {s}", .{ + sub_path, + @errorName(err), + }); + return; }; } + + var tar_file = out_dir.createFile("sources.tar", .{}) catch |err| { + return comp.lockAndSetMiscFailure( + .docs_copy, + "unable to create '{}{s}/sources.tar': {s}", + .{ emit.directory, emit.sub_path, @errorName(err) }, + ); + }; + defer tar_file.close(); + + const root = comp.root_mod.root; + const root_mod_name = comp.root_mod.fully_qualified_name; + const sub_path = if (root.sub_path.len == 0) "." else root.sub_path; + var mod_dir = root.root_dir.handle.openDir(sub_path, .{ .iterate = true }) catch |err| { + return comp.lockAndSetMiscFailure(.docs_copy, "unable to open directory '{}': {s}", .{ + root, @errorName(err), + }); + }; + + var walker = try mod_dir.walk(comp.gpa); + defer walker.deinit(); + + const padding_buffer = [1]u8{0} ** 512; + + while (try walker.next()) |entry| { + switch (entry.kind) { + .file => { + if (!std.mem.endsWith(u8, entry.basename, ".zig")) continue; + if (std.mem.eql(u8, entry.basename, "test.zig")) continue; + if (std.mem.endsWith(u8, entry.basename, "_test.zig")) continue; + }, + else => continue, + } + + var file = mod_dir.openFile(entry.path, .{}) catch |err| { + return comp.lockAndSetMiscFailure(.docs_copy, "unable to open '{}{s}': {s}", .{ + root, entry.path, @errorName(err), + }); + }; + defer file.close(); + + const stat = file.stat() catch |err| { + return comp.lockAndSetMiscFailure(.docs_copy, "unable to stat '{}{s}': {s}", .{ + root, entry.path, @errorName(err), + }); + }; + + var file_header = std.tar.output.Header.init(); + file_header.typeflag = .regular; + try file_header.setPath(root_mod_name, entry.path); + try file_header.setSize(stat.size); + try file_header.updateChecksum(); + + const header_bytes = std.mem.asBytes(&file_header); + const padding = p: { + const remainder = stat.size % 512; + const n = if (remainder > 0) 512 - remainder else 0; + break :p padding_buffer[0..n]; + }; + + var header_and_trailer: [2]std.os.iovec_const = .{ + .{ .iov_base = header_bytes.ptr, .iov_len = header_bytes.len }, + .{ .iov_base = padding.ptr, .iov_len = padding.len }, + }; + + try tar_file.writeFileAll(file, .{ + .in_len = stat.size, + .headers_and_trailers = &header_and_trailer, + .header_count = 1, + }); + } } fn workerDocsWasm(comp: *Compilation, wg: *WaitGroup) void {