commit 12cb5b92851f601c44d48deadea9934146edcffa (tree)
parent 36b65ab59e5e514ad06a11cde96b87c565d86ac8
Author: Andrew Kelley <andrew@ziglang.org>
Date: Sat, 7 Feb 2026 02:46:47 +0100
Merge pull request 'ability to override packages locally' (#31138) from fork-cli into master
Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31138
Diffstat:
6 files changed, 343 insertions(+), 358 deletions(-)
diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig
@@ -1573,6 +1573,23 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
\\ -fsys=[name] Enable a system integration
\\ -fno-sys=[name] Disable a system integration
\\
+ \\ -fdarling, -fno-darling Integration with system-installed Darling to
+ \\ execute macOS programs on Linux hosts
+ \\ (default: no)
+ \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
+ \\ foreign-architecture programs on Linux hosts
+ \\ (default: no)
+ \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc
+ \\ (e.g. glibc or musl) built for multiple foreign
+ \\ architectures, allowing execution of non-native
+ \\ programs that link with libc.
+ \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
+ \\ ARM64 macOS hosts. (default: no)
+ \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
+ \\ execute WASI binaries. (default: no)
+ \\ -fwine, -fno-wine Integration with system-installed Wine to execute
+ \\ Windows programs on Linux hosts. (default: no)
+ \\
\\ Available System Integrations: Enabled:
\\
);
@@ -1592,33 +1609,16 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
try w.writeAll(
\\
\\General Options:
+ \\ -h, --help Print this help and exit
+ \\ -l, --list-steps Print available steps
+ \\
\\ -p, --prefix [path] Where to install files (default: zig-out)
\\ --prefix-lib-dir [path] Where to install libraries
\\ --prefix-exe-dir [path] Where to install executables
\\ --prefix-include-dir [path] Where to install C header files
- \\
\\ --release[=mode] Request release mode, optionally specifying a
\\ preferred optimization mode: fast, safe, small
\\
- \\ -fdarling, -fno-darling Integration with system-installed Darling to
- \\ execute macOS programs on Linux hosts
- \\ (default: no)
- \\ -fqemu, -fno-qemu Integration with system-installed QEMU to execute
- \\ foreign-architecture programs on Linux hosts
- \\ (default: no)
- \\ --libc-runtimes [path] Enhances QEMU integration by providing dynamic libc
- \\ (e.g. glibc or musl) built for multiple foreign
- \\ architectures, allowing execution of non-native
- \\ programs that link with libc.
- \\ -frosetta, -fno-rosetta Rely on Rosetta to execute x86_64 programs on
- \\ ARM64 macOS hosts. (default: no)
- \\ -fwasmtime, -fno-wasmtime Integration with system-installed wasmtime to
- \\ execute WASI binaries. (default: no)
- \\ -fwine, -fno-wine Integration with system-installed Wine to execute
- \\ Windows programs on Linux hosts. (default: no)
- \\
- \\ -h, --help Print this help and exit
- \\ -l, --list-steps Print available steps
\\ --verbose Print commands before executing them
\\ --color [auto|off|on] Enable or disable colored error messages
\\ --error-style [style] Control how build errors are printed
@@ -1641,9 +1641,6 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
\\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss
\\ --test-timeout <timeout> Limit execution time of unit tests, terminating if exceeded.
\\ The timeout must include a unit: ns, us, ms, s, m, h
- \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
- \\ needed (Default) Lazy dependencies are fetched as needed
- \\ all Lazy dependencies are always fetched
\\ --watch Continuously rebuild when source files are modified
\\ --debounce <ms> Delay before rebuilding after changed file detected
\\ --webui[=ip] Enable the web interface on the given IP address
@@ -1656,6 +1653,12 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
\\ -fincremental Enable incremental compilation
\\ -fno-incremental Disable incremental compilation
\\
+ \\Package Management Options:
+ \\ --fetch[=mode] Fetch dependency tree (optionally choose laziness) and exit
+ \\ needed (Default) Lazy dependencies are fetched as needed
+ \\ all Lazy dependencies are always fetched
+ \\ --fork=[path] Override one or more projects from dependency tree
+ \\
\\Advanced Options:
\\ -freference-trace[=num] How many lines of reference trace should be shown per compile error
\\ -fno-reference-trace Disable reference trace
diff --git a/src/Package.zig b/src/Package.zig
@@ -6,10 +6,6 @@ pub const Fetch = @import("Package/Fetch.zig");
pub const build_zig_basename = "build.zig";
pub const Manifest = @import("Package/Manifest.zig");
-pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
-pub const multihash_hex_digest_len = 2 * multihash_len;
-pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
-
pub const Fingerprint = packed struct(u64) {
id: u32,
checksum: u32,
@@ -77,20 +73,6 @@ pub const Hash = struct {
return std.mem.eql(u8, &a.bytes, &b.bytes);
}
- /// Distinguishes whether the legacy multihash format is being stored here.
- pub fn isOld(h: *const Hash) bool {
- if (h.bytes.len < 2) return false;
- const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
- if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
- if (h.toSlice().len != multihash_hex_digest_len) return false;
- return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
- }
-
- test isOld {
- const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
- try std.testing.expect(h.isOld());
- }
-
/// Produces "$name-$semver-$hashplus".
/// * name is the name field from build.zig.zon, asserted to be at most 32
/// bytes and assumed be a valid zig identifier
@@ -137,55 +119,65 @@ pub const Hash = struct {
_ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable;
return result;
}
-};
-pub const MultihashFunction = enum(u16) {
- identity = 0x00,
- sha1 = 0x11,
- @"sha2-256" = 0x12,
- @"sha2-512" = 0x13,
- @"sha3-512" = 0x14,
- @"sha3-384" = 0x15,
- @"sha3-256" = 0x16,
- @"sha3-224" = 0x17,
- @"sha2-384" = 0x20,
- @"sha2-256-trunc254-padded" = 0x1012,
- @"sha2-224" = 0x1013,
- @"sha2-512-224" = 0x1014,
- @"sha2-512-256" = 0x1015,
- @"blake2b-256" = 0xb220,
- _,
-};
+ pub fn projectId(hash: *const Hash) ProjectId {
+ const bytes = hash.toSlice();
+ const name = std.mem.sliceTo(bytes, '-');
+ const encoded_hashplus = bytes[bytes.len - 44 ..];
+ var hashplus: [33]u8 = undefined;
+ std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable;
+ const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little);
+ return .init(name, fingerprint_id);
+ }
-pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
- std.crypto.hash.sha2.Sha256 => .@"sha2-256",
- else => unreachable,
-};
+ test projectId {
+ const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw");
+ const project_id = hash.projectId();
-pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
- const hex_charset = std.fmt.hex_charset;
+ var expected_name: [32]u8 = @splat(0);
+ expected_name[0.."pulseaudio".len].* = "pulseaudio".*;
+ try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
- var result: MultiHashHexDigest = undefined;
+ try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id);
+ }
- result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
- result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
+ test "projectId with dashes in the base64" {
+ const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC");
+ const project_id = hash.projectId();
- result[2] = hex_charset[Hash.Algo.digest_length >> 4];
- result[3] = hex_charset[Hash.Algo.digest_length & 15];
+ var expected_name: [32]u8 = @splat(0);
+ expected_name[0.."dvui".len].* = "dvui".*;
+ try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name);
- for (digest, 0..) |byte, i| {
- result[4 + i * 2] = hex_charset[byte >> 4];
- result[5 + i * 2] = hex_charset[byte & 15];
+ try std.testing.expectEqual(0x99490101, project_id.fingerprint_id);
}
- return result;
-}
+};
-comptime {
- // We avoid unnecessary uleb128 code in hexDigest by asserting here the
- // values are small enough to be contained in the one-byte encoding.
- assert(@intFromEnum(multihash_function) < 127);
- assert(Hash.Algo.digest_length < 127);
-}
+/// Minimum information required to identify whether a package is an artifact
+/// of a given project.
+pub const ProjectId = struct {
+ /// Bytes after name.len are set to zero.
+ padded_name: [32]u8,
+ fingerprint_id: u32,
+
+ pub fn init(name: []const u8, fingerprint_id: u32) ProjectId {
+ var padded_name: [32]u8 = @splat(0);
+ @memcpy(padded_name[0..name.len], name);
+ return .{
+ .padded_name = padded_name,
+ .fingerprint_id = fingerprint_id,
+ };
+ }
+
+ pub fn eql(a: *const ProjectId, b: *const ProjectId) bool {
+ return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name);
+ }
+
+ pub fn hash(a: *const ProjectId) u64 {
+ const x: u64 = @bitCast(a.padded_name[0..8].*);
+ return std.hash.int(x | a.fingerprint_id);
+ }
+};
test Hash {
const example_digest: Hash.Digest = .{
diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig
@@ -76,8 +76,9 @@ use_latest_commit: bool,
/// Relative to the build root of the root package.
package_root: Cache.Path,
error_bundle: ErrorBundle.Wip,
-manifest: ?Manifest,
+manifest: Manifest,
manifest_ast: std.zig.Ast,
+have_manifest: bool,
computed_hash: ComputedHash,
/// Fetch logic notices whether a package has a build.zig file and sets this flag.
has_build_zig: bool,
@@ -142,6 +143,9 @@ pub const JobQueue = struct {
/// Set of hashes that will be additionally fetched even if they are marked
/// as lazy.
unlazy_set: UnlazySet = .{},
+ /// Identifies paths that override all packages in the tree with matching
+ /// project ids.
+ fork_set: ForkSet = .{},
pub const Mode = enum {
/// Non-lazy dependencies are always fetched.
@@ -152,6 +156,38 @@ pub const JobQueue = struct {
};
pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
+ pub const ForkSet = std.ArrayHashMapUnmanaged(Fork, void, Fork.Context, false);
+
+ pub const Fork = struct {
+ path: Cache.Path,
+ manifest_ast: std.zig.Ast,
+ manifest: Package.Manifest,
+ uses: usize,
+
+ pub const Context = struct {
+ pub fn hash(_: @This(), a: Fork) u32 {
+ const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
+ return @truncate(project_id.hash());
+ }
+
+ pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool {
+ const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
+ const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
+ return a_project_id.eql(&b_project_id);
+ }
+ };
+
+ pub const Adapter = struct {
+ pub fn hash(_: @This(), a: Package.ProjectId) u32 {
+ return @truncate(a.hash());
+ }
+
+ pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool {
+ const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
+ return a_project_id.eql(&b_project_id);
+ }
+ };
+ };
pub fn deinit(jq: *JobQueue) void {
const io = jq.io;
@@ -248,7 +284,8 @@ pub const JobQueue = struct {
, .{std.zig.fmtString(hash_slice)});
}
- if (fetch.manifest) |*manifest| {
+ if (fetch.have_manifest) {
+ const manifest = &fetch.manifest;
try buf.appendSlice(
\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
\\
@@ -283,7 +320,8 @@ pub const JobQueue = struct {
);
const root_fetch = jq.all_fetches.items[0];
- const root_manifest = &root_fetch.manifest.?;
+ assert(root_fetch.have_manifest);
+ const root_manifest = &root_fetch.manifest;
for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| {
const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
@@ -536,6 +574,19 @@ pub fn run(f: *Fetch) RunError!void {
var resource_buffer: [init_resource_buffer_size]u8 = undefined;
if (remote.hash) |expected_hash| {
+ const expected_project_id: Package.ProjectId = expected_hash.projectId();
+ if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) |fork| {
+ log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
+ fork.uses += 1;
+ f.package_root = fork.path;
+ f.manifest_ast = fork.manifest_ast;
+ f.manifest = fork.manifest;
+ f.have_manifest = true;
+ try checkBuildFileExistence(f);
+ if (!job_queue.recursive) return;
+ return queueJobsForDeps(f);
+ }
+
const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice());
if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
assert(f.lazy_status != .unavailable);
@@ -673,7 +724,7 @@ fn runResource(
try loadManifest(f, pkg_path);
const filter: Filter = .{
- .include_paths = if (f.manifest) |m| m.paths else .{},
+ .include_paths = if (f.have_manifest) f.manifest.paths else .{},
};
// Ignore errors that were excluded by manifest, such as failure to
@@ -728,21 +779,11 @@ fn runResource(
if (remote_hash) |declared_hash| {
const hash_tok = f.hash_tok.unwrap().?;
- if (declared_hash.isOld()) {
- const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
- if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
- return f.fail(hash_tok, try eb.printString(
- "hash mismatch: manifest declares '{s}' but the fetched package has '{s}'",
- .{ declared_hash.toSlice(), actual_hex },
- ));
- }
- } else {
- if (!computed_package_hash.eql(&declared_hash)) {
- return f.fail(hash_tok, try eb.printString(
- "hash mismatch: manifest declares '{s}' but the fetched package has '{s}'",
- .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
- ));
- }
+ if (!computed_package_hash.eql(&declared_hash)) {
+ return f.fail(hash_tok, try eb.printString(
+ "hash mismatch: manifest declares '{s}' but the fetched package has '{s}'",
+ .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
+ ));
}
} else if (!f.omit_missing_hash_error) {
const notes_len = 1;
@@ -766,7 +807,8 @@ fn runResource(
pub fn computedPackageHash(f: *const Fetch) Package.Hash {
const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
- if (f.manifest) |man| {
+ if (f.have_manifest) {
+ const man = &f.manifest;
var version_buffer: [32]u8 = undefined;
const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer;
return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
@@ -801,45 +843,28 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
const io = f.job_queue.io;
const eb = &f.error_bundle;
const arena = f.arena.allocator();
- const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
+ const manifest_path = try pkg_root.join(arena, Manifest.basename);
+
+ Manifest.load(
io,
- try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }),
arena,
- .limited(Manifest.max_bytes),
- .@"1",
- 0,
+ manifest_path,
+ &f.manifest_ast,
+ eb,
+ &f.manifest,
+ f.allow_missing_paths_field,
) catch |err| switch (err) {
error.FileNotFound => return,
+ error.Canceled => |e| return e,
+ error.ErrorsBundled => return error.FetchFailed,
else => |e| {
- const file_path = try pkg_root.join(arena, Manifest.basename);
try eb.addRootErrorMessage(.{
- .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }),
+ .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
});
return error.FetchFailed;
},
};
-
- const ast = &f.manifest_ast;
- ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
-
- if (ast.errors.len > 0) {
- const file_path = try std.fmt.allocPrint(arena, "{f}" ++ fs.path.sep_str ++ Manifest.basename, .{pkg_root});
- try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, eb);
- return error.FetchFailed;
- }
-
- const rng: std.Random.IoSource = .{ .io = io };
-
- f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{
- .allow_missing_paths_field = f.allow_missing_paths_field,
- });
- const manifest = &f.manifest.?;
-
- if (manifest.errors.len > 0) {
- const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename });
- try manifest.copyErrorsIntoBundle(ast.*, src_path, eb);
- return error.FetchFailed;
- }
+ f.have_manifest = true;
}
fn queueJobsForDeps(f: *Fetch) RunError!void {
@@ -848,7 +873,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
assert(f.job_queue.recursive);
// If the package does not have a build.zig.zon file then there are no dependencies.
- const manifest = f.manifest orelse return;
+ if (!f.have_manifest) return;
+ const manifest = &f.manifest;
const new_fetches, const prog_names = nf: {
const parent_arena = f.arena.allocator();
@@ -953,8 +979,9 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
.package_root = undefined,
.error_bundle = undefined,
- .manifest = null,
+ .manifest = undefined,
.manifest_ast = undefined,
+ .have_manifest = false,
.computed_hash = undefined,
.has_build_zig = false,
.oom_flag = false,
@@ -1931,7 +1958,7 @@ const Filter = struct {
include_paths: std.StringArrayHashMapUnmanaged(void) = .empty,
/// sub_path is relative to the package root.
- pub fn includePath(self: Filter, sub_path: []const u8) bool {
+ pub fn includePath(self: *const Filter, sub_path: []const u8) bool {
if (self.include_paths.count() == 0) return true;
if (self.include_paths.contains("")) return true;
if (self.include_paths.contains(".")) return true;
@@ -2202,206 +2229,6 @@ const UnpackResult = struct {
}
};
-test "set executable bit based on file content" {
- if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;
- const gpa = std.testing.allocator;
- const io = std.testing.io;
-
- var tmp = std.testing.tmpDir(.{});
- defer tmp.cleanup();
-
- const tarball_name = "executables.tar.gz";
- try saveEmbedFile(io, tarball_name, tmp.dir);
- const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name });
- defer gpa.free(tarball_path);
-
- // $ tar -tvf executables.tar.gz
- // drwxrwxr-x 0 executables/
- // -rwxrwxr-x 170 executables/hello
- // lrwxrwxrwx 0 executables/hello_ln -> hello
- // -rw-rw-r-- 0 executables/file1
- // -rw-rw-r-- 17 executables/script_with_shebang_without_exec_bit
- // -rwxrwxr-x 7 executables/script_without_shebang
- // -rwxrwxr-x 17 executables/script
-
- var fb: TestFetchBuilder = undefined;
- var fetch = try fb.build(gpa, io, tmp.dir, tarball_path);
- defer fb.deinit();
-
- try fetch.run();
- try std.testing.expectEqualStrings(
- "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
- &Package.multiHashHexDigest(fetch.computed_hash.digest),
- );
-
- var out = try fb.packageDir();
- defer out.close(io);
- const S = std.posix.S;
- // expect executable bit not set
- try std.testing.expect((try out.statFile(io, "file1", .{})).permissions.toMode() & S.IXUSR == 0);
- try std.testing.expect((try out.statFile(io, "script_without_shebang", .{})).permissions.toMode() & S.IXUSR == 0);
- // expect executable bit set
- try std.testing.expect((try out.statFile(io, "hello", .{})).permissions.toMode() & S.IXUSR != 0);
- try std.testing.expect((try out.statFile(io, "script", .{})).permissions.toMode() & S.IXUSR != 0);
- try std.testing.expect((try out.statFile(io, "script_with_shebang_without_exec_bit", .{})).permissions.toMode() & S.IXUSR != 0);
- try std.testing.expect((try out.statFile(io, "hello_ln", .{})).permissions.toMode() & S.IXUSR != 0);
-
- //
- // $ ls -al zig-cache/tmp/OCz9ovUcstDjTC_U/zig-global-cache/p/1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3
- // -rw-rw-r-- 1 0 Apr file1
- // -rwxrwxr-x 1 170 Apr hello
- // lrwxrwxrwx 1 5 Apr hello_ln -> hello
- // -rwxrwxr-x 1 17 Apr script
- // -rw-rw-r-- 1 7 Apr script_without_shebang
- // -rwxrwxr-x 1 17 Apr script_with_shebang_without_exec_bit
-}
-
-fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void {
- //const tarball_name = "duplicate_paths_excluded.tar.gz";
- const tarball_content = @embedFile("Fetch/testdata/" ++ tarball_name);
- var tmp_file = try dir.createFile(io, tarball_name, .{});
- defer tmp_file.close(io);
- try tmp_file.writeStreamingAll(io, tarball_content);
-}
-
-// Builds Fetch with required dependencies, clears dependencies on deinit().
-const TestFetchBuilder = struct {
- http_client: std.http.Client,
- global_cache_directory: Cache.Directory,
- local_cache_path: Cache.Path,
- job_queue: Fetch.JobQueue,
- fetch: Fetch,
-
- fn build(
- self: *TestFetchBuilder,
- allocator: std.mem.Allocator,
- io: Io,
- cache_parent_dir: std.Io.Dir,
- path_or_url: []const u8,
- ) !*Fetch {
- const global_cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{});
- const package_root_dir = try cache_parent_dir.createDirPathOpen(io, "local-project-root", .{});
-
- self.http_client = .{ .allocator = allocator, .io = io };
- self.global_cache_directory = .{ .handle = global_cache_dir, .path = "zig-global-cache" };
- self.local_cache_path = .{
- .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
- .sub_path = ".zig-cache",
- };
-
- self.job_queue = .{
- .io = io,
- .http_client = &self.http_client,
- .global_cache = self.global_cache_directory,
- .local_cache = self.local_cache_path,
- .root_pkg_path = .{
- .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
- .sub_path = "zig-pkg",
- },
- .recursive = false,
- .read_only = false,
- .debug_hash = false,
- .mode = .needed,
- .prog_node = std.Progress.Node.none,
- };
-
- self.fetch = .{
- .arena = std.heap.ArenaAllocator.init(allocator),
- .location = .{ .path_or_url = path_or_url },
- .location_tok = 0,
- .hash_tok = .none,
- .name_tok = 0,
- .lazy_status = .eager,
- .parent_package_root = .{ .root_dir = .{ .handle = package_root_dir, .path = null } },
- .parent_manifest_ast = null,
- .prog_node = std.Progress.Node.none,
- .job_queue = &self.job_queue,
- .omit_missing_hash_error = true,
- .allow_missing_paths_field = false,
- .use_latest_commit = true,
-
- .package_root = undefined,
- .error_bundle = undefined,
- .manifest = null,
- .manifest_ast = undefined,
- .computed_hash = undefined,
- .has_build_zig = false,
- .oom_flag = false,
- .latest_commit = null,
-
- .module = null,
- };
- return &self.fetch;
- }
-
- fn deinit(self: *TestFetchBuilder) void {
- const io = self.job_queue.io;
- self.fetch.deinit();
- self.job_queue.deinit();
- self.fetch.prog_node.end();
- self.global_cache_directory.handle.close(io);
- self.http_client.deinit();
- }
-
- fn packageDir(self: *TestFetchBuilder) !Io.Dir {
- const io = self.job_queue.io;
- const root = self.fetch.package_root;
- return try root.root_dir.handle.openDir(io, root.sub_path, .{ .iterate = true });
- }
-
- // Test helper, asserts thet package dir constains expected_files.
- // expected_files must be sorted.
- fn expectPackageFiles(self: *TestFetchBuilder, expected_files: []const []const u8) !void {
- const io = self.job_queue.io;
- const gpa = std.testing.allocator;
-
- var package_dir = try self.packageDir();
- defer package_dir.close(io);
-
- var actual_files: std.ArrayList([]u8) = .empty;
- defer actual_files.deinit(gpa);
- defer for (actual_files.items) |file| gpa.free(file);
- var walker = try package_dir.walk(gpa);
- defer walker.deinit();
- while (try walker.next(io)) |entry| {
- if (entry.kind != .file) continue;
- const path = try gpa.dupe(u8, entry.path);
- errdefer gpa.free(path);
- std.mem.replaceScalar(u8, path, std.fs.path.sep, '/');
- try actual_files.append(gpa, path);
- }
- std.mem.sortUnstable([]u8, actual_files.items, {}, struct {
- fn lessThan(_: void, a: []u8, b: []u8) bool {
- return std.mem.lessThan(u8, a, b);
- }
- }.lessThan);
-
- try std.testing.expectEqual(expected_files.len, actual_files.items.len);
- for (expected_files, 0..) |file_name, i| {
- try std.testing.expectEqualStrings(file_name, actual_files.items[i]);
- }
- try std.testing.expectEqualDeep(expected_files, actual_files.items);
- }
-
- // Test helper, asserts that fetch has failed with `msg` error message.
- fn expectFetchErrors(self: *TestFetchBuilder, notes_len: usize, msg: []const u8) !void {
- const gpa = std.testing.allocator;
-
- var errors = try self.fetch.error_bundle.toOwnedBundle("");
- defer errors.deinit(gpa);
-
- const em = errors.getErrorMessage(errors.getMessages()[0]);
- try std.testing.expectEqual(1, em.count);
- if (notes_len > 0) {
- try std.testing.expectEqual(notes_len, em.notes_len);
- }
- var aw: Io.Writer.Allocating = .init(gpa);
- defer aw.deinit();
- try errors.renderToWriter(.{}, &aw.writer);
- try std.testing.expectEqualStrings(msg, aw.written());
- }
-};
-
test {
_ = Filter;
_ = FileType;
diff --git a/src/Package/Fetch/testdata/executables.tar.gz b/src/Package/Fetch/testdata/executables.tar.gz
Binary files differ.
diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig
@@ -1,10 +1,13 @@
const Manifest = @This();
+
const std = @import("std");
+const Io = std.Io;
const mem = std.mem;
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const Ast = std.zig.Ast;
const testing = std.testing;
+
const Package = @import("../Package.zig");
pub const max_bytes = 10 * 1024 * 1024;
@@ -53,7 +56,7 @@ pub const ParseOptions = struct {
pub const Error = Allocator.Error;
-pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
+pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest {
const main_node_index = ast.nodeData(.root).node;
var arena_instance = std.heap.ArenaAllocator.init(gpa);
@@ -61,7 +64,7 @@ pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) E
var p: Parse = .{
.gpa = gpa,
- .ast = ast,
+ .ast = ast.*,
.arena = arena_instance.allocator(),
.errors = .{},
@@ -578,6 +581,45 @@ const Parse = struct {
}
};
+pub fn load(
+ io: Io,
+ arena: Allocator,
+ manifest_path: std.Build.Cache.Path,
+ ast: *std.zig.Ast,
+ error_bundle: *std.zig.ErrorBundle.Wip,
+ manifest: *Manifest,
+ allow_missing_paths_field: bool,
+) !void {
+ const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions(
+ io,
+ manifest_path.sub_path,
+ arena,
+ .limited(max_bytes),
+ .@"1",
+ 0,
+ );
+
+ ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
+
+ if (ast.errors.len > 0) {
+ const file_path = try manifest_path.joinString(arena, "");
+ try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle);
+ return error.ErrorsBundled;
+ }
+
+ const rng: std.Random.IoSource = .{ .io = io };
+
+ manifest.* = try parse(arena, ast, rng.interface(), .{
+ .allow_missing_paths_field = allow_missing_paths_field,
+ });
+
+ if (manifest.errors.len > 0) {
+ const src_path = try error_bundle.printString("{f}", .{manifest_path});
+ try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle);
+ return error.ErrorsBundled;
+ }
+}
+
test "basic" {
const gpa = testing.allocator;
@@ -603,7 +645,7 @@ test "basic" {
var rng = std.Random.DefaultPrng.init(0);
- var manifest = try Manifest.parse(gpa, ast, rng.random(), .{});
+ var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
defer manifest.deinit(gpa);
try testing.expect(manifest.errors.len == 0);
@@ -649,7 +691,7 @@ test "minimum_zig_version" {
var rng = std.Random.DefaultPrng.init(0);
- var manifest = try Manifest.parse(gpa, ast, rng.random(), .{});
+ var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
defer manifest.deinit(gpa);
try testing.expect(manifest.errors.len == 0);
@@ -684,7 +726,7 @@ test "minimum_zig_version - invalid version" {
var rng = std.Random.DefaultPrng.init(0);
- var manifest = try Manifest.parse(gpa, ast, rng.random(), .{});
+ var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{});
defer manifest.deinit(gpa);
try testing.expect(manifest.errors.len == 1);
diff --git a/src/main.zig b/src/main.zig
@@ -4896,7 +4896,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
var override_build_runner: ?[]const u8 = EnvVar.ZIG_BUILD_RUNNER.get(environ_map);
- var child_argv = std.array_list.Managed([]const u8).init(arena);
+ var child_argv: std.ArrayList([]const u8) = .empty;
+ var forks: std.ArrayList(Fork) = .empty;
var reference_trace: ?u32 = null;
var debug_compile_errors = false;
var verbose_link = (native_os != .wasi or builtin.link_libc) and
@@ -4917,24 +4918,24 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
var debug_libc_paths_file: ?[]const u8 = null;
const argv_index_exe = child_argv.items.len;
- _ = try child_argv.addOne();
+ _ = try child_argv.addOne(arena);
const self_exe_path = try process.executablePathAlloc(io, arena);
- try child_argv.append(self_exe_path);
+ try child_argv.append(arena, self_exe_path);
const argv_index_zig_lib_dir = child_argv.items.len;
- _ = try child_argv.addOne();
+ _ = try child_argv.addOne(arena);
const argv_index_build_file = child_argv.items.len;
- _ = try child_argv.addOne();
+ _ = try child_argv.addOne(arena);
const argv_index_cache_dir = child_argv.items.len;
- _ = try child_argv.addOne();
+ _ = try child_argv.addOne(arena);
const argv_index_global_cache_dir = child_argv.items.len;
- _ = try child_argv.addOne();
+ _ = try child_argv.addOne(arena);
- try child_argv.appendSlice(&.{
+ try child_argv.appendSlice(arena, &.{
"--seed",
try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}),
});
@@ -4955,7 +4956,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
// read this file in the parent to obtain the results, in the case the child
// exits with code 3.
const results_tmp_file_nonce = std.fmt.hex(randInt(io, u64));
- try child_argv.append("-Z" ++ results_tmp_file_nonce);
+ try child_argv.append(arena, "-Z" ++ results_tmp_file_nonce);
var color: Color = .auto;
var n_jobs: ?u32 = null;
@@ -5000,11 +5001,24 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
fatal("expected [needed|all] after '--fetch=', found '{s}'", .{
sub_arg,
});
+ } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| {
+ try forks.append(arena, .{
+ .manifest_ast = undefined,
+ .manifest = undefined,
+ .error_bundle = undefined,
+ .arena_allocator = undefined,
+ .path = .{
+ .root_dir = .cwd(),
+ .sub_path = sub_arg,
+ },
+ .failed = false,
+ });
+ continue;
} else if (mem.eql(u8, arg, "--system")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
i += 1;
system_pkg_dir_path = args[i];
- try child_argv.append("--system");
+ try child_argv.append(arena, "--system");
continue;
} else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| {
reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
@@ -5014,7 +5028,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
reference_trace = null;
} else if (mem.eql(u8, arg, "--debug-log")) {
if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
- try child_argv.appendSlice(args[i .. i + 2]);
+ try child_argv.appendSlice(arena, args[i .. i + 2]);
i += 1;
if (!build_options.enable_logging) {
warn("Zig was compiled without logging enabled (-Dlog). --debug-log has no effect.", .{});
@@ -5070,7 +5084,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
color = std.meta.stringToEnum(Color, args[i]) orelse {
fatal("expected [auto|on|off] after {s}, found '{s}'", .{ arg, args[i] });
};
- try child_argv.appendSlice(&.{ arg, args[i] });
+ try child_argv.appendSlice(arena, &.{ arg, args[i] });
continue;
} else if (mem.cutPrefix(u8, arg, "-j")) |str| {
const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| {
@@ -5090,11 +5104,11 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
} else if (mem.eql(u8, arg, "--")) {
// The rest of the args are supposed to get passed onto
// build runner's `build.args`
- try child_argv.appendSlice(args[i..]);
+ try child_argv.appendSlice(arena, args[i..]);
break;
}
}
- try child_argv.append(arg);
+ try child_argv.append(arena, arg);
}
}
@@ -5182,6 +5196,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
defer http_client.deinit();
var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{};
+ var fork_set: Package.Fetch.JobQueue.ForkSet = .{};
+
+ {
+ // Populate fork_set.
+ var group: Io.Group = .init;
+ defer group.cancel(io);
+
+ for (forks.items) |*fork|
+ group.async(io, Fork.load, .{ io, gpa, fork, color });
+
+ try group.await(io);
+
+ for (forks.items) |*fork| {
+ if (fork.failed) process.exit(1);
+ try fork_set.put(arena, .{
+ .path = fork.path,
+ .manifest_ast = fork.manifest_ast,
+ .manifest = fork.manifest,
+ .uses = 0,
+ }, {});
+ }
+ }
+ defer Fork.deinitList(forks.items);
// This loop is re-evaluated when the build script exits with an indication that it
// could not continue due to missing lazy dependencies.
@@ -5235,6 +5272,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);
defer fetch_prog_node.end();
+ // Reset fork match counts.
+ for (fork_set.keys()) |*fork| fork.uses = 0;
+
var job_queue: Package.Fetch.JobQueue = .{
.io = io,
.http_client = &http_client,
@@ -5245,6 +5285,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.recursive = true,
.debug_hash = false,
.unlazy_set = unlazy_set,
+ .fork_set = fork_set,
.mode = fetch_mode,
.prog_node = fetch_prog_node,
};
@@ -5290,8 +5331,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
.package_root = undefined,
.error_bundle = undefined,
- .manifest = null,
+ .manifest = undefined,
.manifest_ast = undefined,
+ .have_manifest = false,
.computed_hash = undefined,
.has_build_zig = true,
.oom_flag = false,
@@ -5309,6 +5351,26 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" });
try job_queue.group.await(io);
+ {
+ // Ensure that forks were actually used. This is done
+ // before printing manifest errors because using a fork can
+ // prevent them.
+ var any_unused = false;
+ for (fork_set.keys()) |*fork| {
+ if (fork.uses == 0) {
+ std.log.err("fork {f} matched no {s} packages", .{
+ fork.path, fork.manifest.name,
+ });
+ any_unused = true;
+ } else {
+ std.log.info("fork {f} matched {d} {s} packages", .{
+ fork.path, fork.uses, fork.manifest.name,
+ });
+ }
+ }
+ if (any_unused) process.exit(1);
+ }
+
try job_queue.consolidateErrors();
if (fetch.error_bundle.root_list.items.len > 0) {
@@ -5369,7 +5431,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
// dependencies' build.zig modules by name.
for (fetches) |f| {
const mod = f.module orelse continue;
- const man = f.manifest orelse continue;
+ if (!f.have_manifest) continue;
+ const man = &f.manifest;
const dep_names = man.dependencies.keys();
try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len));
for (dep_names, man.dependencies.values()) |name, dep| {
@@ -5518,6 +5581,63 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
}
}
+const Fork = struct {
+ path: Path,
+ manifest_ast: std.zig.Ast,
+ manifest: Package.Manifest,
+ error_bundle: std.zig.ErrorBundle.Wip,
+ failed: bool,
+ arena_allocator: std.heap.ArenaAllocator,
+
+ fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void {
+ loadFallible(io, gpa, fork, color) catch |err| switch (err) {
+ error.Canceled => |e| return e,
+ error.AlreadyReported => fork.failed = true,
+ else => |e| {
+ std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e });
+ fork.failed = true;
+ },
+ };
+ }
+
+ fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void {
+ fork.arena_allocator = .init(gpa);
+ const arena = fork.arena_allocator.allocator();
+
+ var error_bundle: std.zig.ErrorBundle.Wip = undefined;
+ try error_bundle.init(gpa);
+ defer error_bundle.deinit();
+
+ const manifest_path = try fork.path.join(arena, Package.Manifest.basename);
+
+ Package.Manifest.load(
+ io,
+ arena,
+ manifest_path,
+ &fork.manifest_ast,
+ &error_bundle,
+ &fork.manifest,
+ true,
+ ) catch |err| switch (err) {
+ error.Canceled => |e| return e,
+ error.ErrorsBundled => {
+ assert(error_bundle.root_list.items.len > 0);
+ var errors = try error_bundle.toOwnedBundle("");
+ errors.renderToStderr(io, .{}, color) catch {};
+ return error.AlreadyReported;
+ },
+ else => |e| {
+ std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e });
+ return error.AlreadyReported;
+ },
+ };
+ }
+
+ fn deinitList(forks: []Fork) void {
+ for (forks) |*fork| fork.arena_allocator.deinit();
+ }
+};
+
const JitCmdOptions = struct {
cmd_name: []const u8,
root_src_path: []const u8,
@@ -7048,8 +7168,9 @@ fn cmdFetch(
.package_root = undefined,
.error_bundle = undefined,
- .manifest = null,
+ .manifest = undefined,
.manifest_ast = undefined,
+ .have_manifest = false,
.computed_hash = undefined,
.has_build_zig = false,
.oom_flag = false,
@@ -7085,9 +7206,9 @@ fn cmdFetch(
},
.yes, .exact => |name| name: {
if (name) |n| break :name n;
- const fetched_manifest = fetch.manifest orelse
+ if (!fetch.have_manifest)
fatal("unable to determine name; fetched package has no build.zig.zon file", .{});
- break :name fetched_manifest.name;
+ break :name fetch.manifest.name;
},
};
@@ -7410,7 +7531,7 @@ fn loadManifest(
process.exit(2);
}
- var manifest = try Package.Manifest.parse(gpa, ast, rng.interface(), .{});
+ var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{});
errdefer manifest.deinit(gpa);
if (manifest.errors.len > 0) {
