commit a9e1cf3049d1d90d7e62950c3b7f4b088081caa8 (tree)
parent fc48467a97021cb872ff2a947f96e882274c39c1
Author: Andrew Kelley <andrew@ziglang.org>
Date: Mon, 13 Feb 2023 17:23:19 -0500
Merge pull request #14571 from ziglang/more-build-zig
std.Build.ConfigHeaderStep: support sentinel-terminated strings
Diffstat:
25 files changed, 1642 insertions(+), 1466 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -216,6 +216,9 @@ set(ZIG_STAGE2_SOURCES
"${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
"${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
"${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
+ "${CMAKE_SOURCE_DIR}/lib/std/Build.zig"
+ "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig"
+ "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache/DepTokenizer.zig"
"${CMAKE_SOURCE_DIR}/lib/std/builtin.zig"
"${CMAKE_SOURCE_DIR}/lib/std/c.zig"
"${CMAKE_SOURCE_DIR}/lib/std/c/linux.zig"
@@ -523,9 +526,7 @@ set(ZIG_STAGE2_SOURCES
"${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig"
"${CMAKE_SOURCE_DIR}/src/Air.zig"
"${CMAKE_SOURCE_DIR}/src/AstGen.zig"
- "${CMAKE_SOURCE_DIR}/src/Cache.zig"
"${CMAKE_SOURCE_DIR}/src/Compilation.zig"
- "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
"${CMAKE_SOURCE_DIR}/src/Liveness.zig"
"${CMAKE_SOURCE_DIR}/src/Module.zig"
"${CMAKE_SOURCE_DIR}/src/Package.zig"
diff --git a/build.zig b/build.zig
@@ -40,15 +40,11 @@ pub fn build(b: *std.Build) !void {
});
docgen_exe.single_threaded = single_threaded;
- const rel_zig_exe = try fs.path.relative(b.allocator, b.build_root, b.zig_exe);
- const langref_out_path = fs.path.join(
- b.allocator,
- &[_][]const u8{ b.cache_root, "langref.html" },
- ) catch unreachable;
+ const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"});
const docgen_cmd = docgen_exe.run();
docgen_cmd.addArgs(&[_][]const u8{
"--zig",
- rel_zig_exe,
+ b.zig_exe,
"doc" ++ fs.path.sep_str ++ "langref.html.in",
langref_out_path,
});
@@ -215,7 +211,7 @@ pub fn build(b: *std.Build) !void {
var code: u8 = undefined;
const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{
- "git", "-C", b.build_root, "describe", "--match", "*.*.*", "--tags",
+ "git", "-C", b.build_root.path orelse ".", "describe", "--match", "*.*.*", "--tags",
}, &code, .Ignore) catch {
break :v version_string;
};
diff --git a/lib/build_runner.zig b/lib/build_runner.zig
@@ -43,13 +43,40 @@ pub fn main() !void {
const host = try std.zig.system.NativeTargetInfo.detect(.{});
+ const build_root_directory: std.Build.Cache.Directory = .{
+ .path = build_root,
+ .handle = try std.fs.cwd().openDir(build_root, .{}),
+ };
+
+ const local_cache_directory: std.Build.Cache.Directory = .{
+ .path = cache_root,
+ .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}),
+ };
+
+ const global_cache_directory: std.Build.Cache.Directory = .{
+ .path = global_cache_root,
+ .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}),
+ };
+
+ var cache: std.Build.Cache = .{
+ .gpa = allocator,
+ .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
+ cache.addPrefix(build_root_directory);
+ cache.addPrefix(local_cache_directory);
+ cache.addPrefix(global_cache_directory);
+
+ //cache.hash.addBytes(builtin.zig_version);
+
const builder = try std.Build.create(
allocator,
zig_exe,
- build_root,
- cache_root,
- global_cache_root,
+ build_root_directory,
+ local_cache_directory,
+ global_cache_directory,
host,
+ &cache,
);
defer builder.destroy();
@@ -138,7 +165,7 @@ pub fn main() !void {
return usageAndErr(builder, false, stderr_stream);
};
} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
- builder.override_lib_dir = nextArg(args, &arg_idx) orelse {
+ builder.zig_lib_dir = nextArg(args, &arg_idx) orelse {
std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});
return usageAndErr(builder, false, stderr_stream);
};
diff --git a/lib/std/Build.zig b/lib/std/Build.zig
@@ -19,6 +19,8 @@ const NativeTargetInfo = std.zig.system.NativeTargetInfo;
const Sha256 = std.crypto.hash.sha2.Sha256;
const Build = @This();
+pub const Cache = @import("Build/Cache.zig");
+
/// deprecated: use `CompileStep`.
pub const LibExeObjStep = CompileStep;
/// deprecated: use `Build`.
@@ -77,11 +79,12 @@ search_prefixes: ArrayList([]const u8),
libc_file: ?[]const u8 = null,
installed_files: ArrayList(InstalledFile),
/// Path to the directory containing build.zig.
-build_root: []const u8,
-cache_root: []const u8,
-global_cache_root: []const u8,
-/// zig lib dir
-override_lib_dir: ?[]const u8,
+build_root: Cache.Directory,
+cache_root: Cache.Directory,
+global_cache_root: Cache.Directory,
+cache: *Cache,
+/// If non-null, overrides the default zig lib dir.
+zig_lib_dir: ?[]const u8,
vcpkg_root: VcpkgRoot = .unattempted,
pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
args: ?[][]const u8 = null,
@@ -185,10 +188,11 @@ pub const DirList = struct {
pub fn create(
allocator: Allocator,
zig_exe: []const u8,
- build_root: []const u8,
- cache_root: []const u8,
- global_cache_root: []const u8,
+ build_root: Cache.Directory,
+ cache_root: Cache.Directory,
+ global_cache_root: Cache.Directory,
host: NativeTargetInfo,
+ cache: *Cache,
) !*Build {
const env_map = try allocator.create(EnvMap);
env_map.* = try process.getEnvMap(allocator);
@@ -197,8 +201,9 @@ pub fn create(
self.* = Build{
.zig_exe = zig_exe,
.build_root = build_root,
- .cache_root = try fs.path.relative(allocator, build_root, cache_root),
+ .cache_root = cache_root,
.global_cache_root = global_cache_root,
+ .cache = cache,
.verbose = false,
.verbose_link = false,
.verbose_cc = false,
@@ -230,7 +235,7 @@ pub fn create(
.step = Step.init(.top_level, "uninstall", allocator, makeUninstall),
.description = "Remove build artifacts from prefix path",
},
- .override_lib_dir = null,
+ .zig_lib_dir = null,
.install_path = undefined,
.args = null,
.host = host,
@@ -245,7 +250,7 @@ pub fn create(
fn createChild(
parent: *Build,
dep_name: []const u8,
- build_root: []const u8,
+ build_root: Cache.Directory,
args: anytype,
) !*Build {
const child = try createChildOnly(parent, dep_name, build_root);
@@ -253,7 +258,7 @@ fn createChild(
return child;
}
-fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) !*Build {
+fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build {
const allocator = parent.allocator;
const child = try allocator.create(Build);
child.* = .{
@@ -297,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8)
.build_root = build_root,
.cache_root = parent.cache_root,
.global_cache_root = parent.global_cache_root,
- .override_lib_dir = parent.override_lib_dir,
+ .cache = parent.cache,
+ .zig_lib_dir = parent.zig_lib_dir,
.debug_log_scopes = parent.debug_log_scopes,
.debug_compile_errors = parent.debug_compile_errors,
.enable_darling = parent.enable_darling,
@@ -348,7 +354,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
.used = false,
});
},
- .Enum => {
+ .Enum, .EnumLiteral => {
try b.user_input_options.put(field.name, .{
.name = field.name,
.value = .{ .scalar = @tagName(v) },
@@ -379,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
_ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch
unreachable;
- const install_prefix = b.pathJoin(&.{ b.cache_root, "i", &hash_basename });
+ const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &hash_basename });
b.resolveInstallPrefix(install_prefix, .{});
}
@@ -396,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:
self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix });
} else {
self.install_prefix = install_prefix orelse
- (self.pathJoin(&.{ self.build_root, "zig-out" }));
+ (self.build_root.join(self.allocator, &.{"zig-out"}) catch @panic("unhandled error"));
self.install_path = self.install_prefix;
}
@@ -599,6 +605,28 @@ pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep {
return run_step;
}
+/// Creates a `RunStep` with an executable built with `addExecutable`.
+/// Add command line arguments with methods of `RunStep`.
+pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep {
+ assert(exe.kind == .exe or exe.kind == .test_exe);
+
+ // It doesn't have to be native. We catch that if you actually try to run it.
+ // Consider that this is declarative; the run step may not be run unless a user
+ // option is supplied.
+ const run_step = RunStep.create(b, b.fmt("run {s}", .{exe.step.name}));
+ run_step.addArtifactArg(exe);
+
+ if (exe.kind == .test_exe) {
+ run_step.addArg(b.zig_exe);
+ }
+
+ if (exe.vcpkg_bin_path) |path| {
+ run_step.addPathDir(path);
+ }
+
+ return run_step;
+}
+
/// Using the `values` provided, produces a C header file, possibly based on a
/// template input file (e.g. config.h.in).
/// When an input template file is provided, this function will fail the build
@@ -674,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS
}
pub fn make(self: *Build, step_names: []const []const u8) !void {
- try self.makePath(self.cache_root);
-
var wanted_steps = ArrayList(*Step).init(self.allocator);
defer wanted_steps.deinit();
@@ -1201,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap,
}
}
-pub fn makePath(self: *Build, path: []const u8) !void {
- fs.cwd().makePath(self.pathFromRoot(path)) catch |err| {
- log.err("Unable to create path {s}: {s}", .{ path, @errorName(err) });
- return err;
- };
-}
-
pub fn installArtifact(self: *Build, artifact: *CompileStep) void {
self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);
}
@@ -1322,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void {
src_file.close();
}
-pub fn pathFromRoot(self: *Build, rel_path: []const u8) []u8 {
- return fs.path.resolve(self.allocator, &[_][]const u8{ self.build_root, rel_path }) catch @panic("OOM");
+pub fn pathFromRoot(b: *Build, p: []const u8) []u8 {
+ return fs.path.resolve(b.allocator, &.{ b.build_root.path orelse ".", p }) catch @panic("OOM");
}
pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 {
@@ -1544,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
fn dependencyInner(
b: *Build,
name: []const u8,
- build_root: []const u8,
+ build_root_string: []const u8,
comptime build_zig: type,
args: anytype,
) *Dependency {
+ const build_root: std.Build.Cache.Directory = .{
+ .path = build_root_string,
+ .handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| {
+ std.debug.print("unable to open '{s}': {s}\n", .{
+ build_root_string, @errorName(err),
+ });
+ std.process.exit(1);
+ },
+ };
const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error");
sub_builder.runBuild(build_zig) catch @panic("unhandled error");
@@ -1568,26 +1596,6 @@ pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void {
}
}
-test "builder.findProgram compiles" {
- if (builtin.os.tag == .wasi) return error.SkipZigTest;
-
- var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
- defer arena.deinit();
-
- const host = try NativeTargetInfo.detect(.{});
-
- const builder = try Build.create(
- arena.allocator(),
- "zig",
- "zig-cache",
- "zig-cache",
- "zig-cache",
- host,
- );
- defer builder.destroy();
- _ = builder.findProgram(&[_][]const u8{}, &[_][]const u8{}) catch null;
-}
-
pub const Module = struct {
builder: *Build,
/// This could either be a generated file, in which case the module
@@ -1616,7 +1624,6 @@ pub const GeneratedFile = struct {
};
/// A file source is a reference to an existing or future file.
-///
pub const FileSource = union(enum) {
/// A plain file path, relative to build root or absolute.
path: []const u8,
diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig
@@ -0,0 +1,1253 @@
+//! Manages `zig-cache` directories.
+//! This is not a general-purpose cache. It is designed to be fast and simple,
+//! not to withstand attacks using specially-crafted input.
+
+pub const Directory = struct {
+ /// This field is redundant for operations that can act on the open directory handle
+ /// directly, but it is needed when passing the directory to a child process.
+ /// `null` means cwd.
+ path: ?[]const u8,
+ handle: std.fs.Dir,
+
+ pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
+ if (self.path) |p| {
+ // TODO clean way to do this with only 1 allocation
+ const part2 = try std.fs.path.join(allocator, paths);
+ defer allocator.free(part2);
+ return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
+ } else {
+ return std.fs.path.join(allocator, paths);
+ }
+ }
+
+ pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
+ if (self.path) |p| {
+ // TODO clean way to do this with only 1 allocation
+ const part2 = try std.fs.path.join(allocator, paths);
+ defer allocator.free(part2);
+ return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
+ } else {
+ return std.fs.path.joinZ(allocator, paths);
+ }
+ }
+
+ /// Whether or not the handle should be closed, or the path should be freed
+ /// is determined by usage, however this function is provided for convenience
+ /// if it happens to be what the caller needs.
+ pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
+ self.handle.close();
+ if (self.path) |p| gpa.free(p);
+ self.* = undefined;
+ }
+};
+
+gpa: Allocator,
+manifest_dir: fs.Dir,
+hash: HashHelper = .{},
+/// This value is accessed from multiple threads, protected by mutex.
+recent_problematic_timestamp: i128 = 0,
+mutex: std.Thread.Mutex = .{},
+
+/// A set of strings such as the zig library directory or project source root, which
+/// are stripped from the file paths before putting into the cache. They
+/// are replaced with single-character indicators. This is not to save
+/// space but to eliminate absolute file paths. This improves portability
+/// and usefulness of the cache for advanced use cases.
+prefixes_buffer: [4]Directory = undefined,
+prefixes_len: usize = 0,
+
+pub const DepTokenizer = @import("Cache/DepTokenizer.zig");
+
+const Cache = @This();
+const std = @import("std");
+const builtin = @import("builtin");
+const crypto = std.crypto;
+const fs = std.fs;
+const assert = std.debug.assert;
+const testing = std.testing;
+const mem = std.mem;
+const fmt = std.fmt;
+const Allocator = std.mem.Allocator;
+const log = std.log.scoped(.cache);
+
+pub fn addPrefix(cache: *Cache, directory: Directory) void {
+ cache.prefixes_buffer[cache.prefixes_len] = directory;
+ cache.prefixes_len += 1;
+}
+
+/// Be sure to call `Manifest.deinit` after successful initialization.
+pub fn obtain(cache: *Cache) Manifest {
+ return Manifest{
+ .cache = cache,
+ .hash = cache.hash,
+ .manifest_file = null,
+ .manifest_dirty = false,
+ .hex_digest = undefined,
+ };
+}
+
+pub fn prefixes(cache: *const Cache) []const Directory {
+ return cache.prefixes_buffer[0..cache.prefixes_len];
+}
+
+const PrefixedPath = struct {
+ prefix: u8,
+ sub_path: []u8,
+};
+
+fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
+ const gpa = cache.gpa;
+ const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
+ errdefer gpa.free(resolved_path);
+ return findPrefixResolved(cache, resolved_path);
+}
+
+/// Takes ownership of `resolved_path` on success.
+fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
+ const gpa = cache.gpa;
+ const prefixes_slice = cache.prefixes();
+ var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
+ while (i < prefixes_slice.len) : (i += 1) {
+ const p = prefixes_slice[i].path.?;
+ if (mem.startsWith(u8, resolved_path, p)) {
+ // +1 to skip over the path separator here
+ const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
+ gpa.free(resolved_path);
+ return PrefixedPath{
+ .prefix = @intCast(u8, i),
+ .sub_path = sub_path,
+ };
+ }
+ }
+
+ return PrefixedPath{
+ .prefix = 0,
+ .sub_path = resolved_path,
+ };
+}
+
+/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
+pub const bin_digest_len = 16;
+pub const hex_digest_len = bin_digest_len * 2;
+pub const BinDigest = [bin_digest_len]u8;
+
+const manifest_file_size_max = 50 * 1024 * 1024;
+
+/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
+/// provides enough collision resistance for the Manifest use cases, while being one of our
+/// fastest options right now.
+pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
+
+/// Initial state, that can be copied.
+pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
+
+pub const File = struct {
+ prefixed_path: ?PrefixedPath,
+ max_file_size: ?usize,
+ stat: Stat,
+ bin_digest: BinDigest,
+ contents: ?[]const u8,
+
+ pub const Stat = struct {
+ inode: fs.File.INode,
+ size: u64,
+ mtime: i128,
+ };
+
+ pub fn deinit(self: *File, gpa: Allocator) void {
+ if (self.prefixed_path) |pp| {
+ gpa.free(pp.sub_path);
+ self.prefixed_path = null;
+ }
+ if (self.contents) |contents| {
+ gpa.free(contents);
+ self.contents = null;
+ }
+ self.* = undefined;
+ }
+};
+
+pub const HashHelper = struct {
+ hasher: Hasher = hasher_init,
+
+ /// Record a slice of bytes as an dependency of the process being cached
+ pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
+ hh.hasher.update(mem.asBytes(&bytes.len));
+ hh.hasher.update(bytes);
+ }
+
+ pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
+ hh.add(optional_bytes != null);
+ hh.addBytes(optional_bytes orelse return);
+ }
+
+ pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
+ hh.add(list_of_bytes.len);
+ for (list_of_bytes) |bytes| hh.addBytes(bytes);
+ }
+
+ /// Convert the input value into bytes and record it as a dependency of the process being cached.
+ pub fn add(hh: *HashHelper, x: anytype) void {
+ switch (@TypeOf(x)) {
+ std.builtin.Version => {
+ hh.add(x.major);
+ hh.add(x.minor);
+ hh.add(x.patch);
+ },
+ std.Target.Os.TaggedVersionRange => {
+ switch (x) {
+ .linux => |linux| {
+ hh.add(linux.range.min);
+ hh.add(linux.range.max);
+ hh.add(linux.glibc);
+ },
+ .windows => |windows| {
+ hh.add(windows.min);
+ hh.add(windows.max);
+ },
+ .semver => |semver| {
+ hh.add(semver.min);
+ hh.add(semver.max);
+ },
+ .none => {},
+ }
+ },
+ else => switch (@typeInfo(@TypeOf(x))) {
+ .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
+ else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
+ },
+ }
+ }
+
+ pub fn addOptional(hh: *HashHelper, optional: anytype) void {
+ hh.add(optional != null);
+ hh.add(optional orelse return);
+ }
+
+ /// Returns a hex encoded hash of the inputs, without modifying state.
+ pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
+ var copy = hh;
+ return copy.final();
+ }
+
+ pub fn peekBin(hh: HashHelper) BinDigest {
+ var copy = hh;
+ var bin_digest: BinDigest = undefined;
+ copy.hasher.final(&bin_digest);
+ return bin_digest;
+ }
+
+ /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
+ pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
+ var bin_digest: BinDigest = undefined;
+ hh.hasher.final(&bin_digest);
+
+ var out_digest: [hex_digest_len]u8 = undefined;
+ _ = std.fmt.bufPrint(
+ &out_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&bin_digest)},
+ ) catch unreachable;
+ return out_digest;
+ }
+};
+
+pub const Lock = struct {
+ manifest_file: fs.File,
+
+ pub fn release(lock: *Lock) void {
+ if (builtin.os.tag == .windows) {
+ // Windows does not guarantee that locks are immediately unlocked when
+ // the file handle is closed. See LockFileEx documentation.
+ lock.manifest_file.unlock();
+ }
+
+ lock.manifest_file.close();
+ lock.* = undefined;
+ }
+};
+
+pub const Manifest = struct {
+ cache: *Cache,
+ /// Current state for incremental hashing.
+ hash: HashHelper,
+ manifest_file: ?fs.File,
+ manifest_dirty: bool,
+ /// Set this flag to true before calling hit() in order to indicate that
+ /// upon a cache hit, the code using the cache will not modify the files
+ /// within the cache directory. This allows multiple processes to utilize
+ /// the same cache directory at the same time.
+ want_shared_lock: bool = true,
+ have_exclusive_lock: bool = false,
+ // Indicate that we want isProblematicTimestamp to perform a filesystem write in
+ // order to obtain a problematic timestamp for the next call. Calls after that
+ // will then use the same timestamp, to avoid unnecessary filesystem writes.
+ want_refresh_timestamp: bool = true,
+ files: std.ArrayListUnmanaged(File) = .{},
+ hex_digest: [hex_digest_len]u8,
+ /// Populated when hit() returns an error because of one
+ /// of the files listed in the manifest.
+ failed_file_index: ?usize = null,
+ /// Keeps track of the last time we performed a file system write to observe
+ /// what time the file system thinks it is, according to its own granularity.
+ recent_problematic_timestamp: i128 = 0,
+
+ /// Add a file as a dependency of process being cached. When `hit` is
+ /// called, the file's contents will be checked to ensure that it matches
+ /// the contents from previous times.
+ ///
+ /// Max file size will be used to determine the amount of space the file contents
+ /// are allowed to take up in memory. If max_file_size is null, then the contents
+ /// will not be loaded into memory.
+ ///
+ /// Returns the index of the entry in the `files` array list. You can use it
+ /// to access the contents of the file after calling `hit()` like so:
+ ///
+ /// ```
+ /// var file_contents = cache_hash.files.items[file_index].contents.?;
+ /// ```
+ pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
+ assert(self.manifest_file == null);
+
+ const gpa = self.cache.gpa;
+ try self.files.ensureUnusedCapacity(gpa, 1);
+ const prefixed_path = try self.cache.findPrefix(file_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ self.files.addOneAssumeCapacity().* = .{
+ .prefixed_path = prefixed_path,
+ .contents = null,
+ .max_file_size = max_file_size,
+ .stat = undefined,
+ .bin_digest = undefined,
+ };
+
+ self.hash.add(prefixed_path.prefix);
+ self.hash.addBytes(prefixed_path.sub_path);
+
+ return self.files.items.len - 1;
+ }
+
+ pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
+ self.hash.add(optional_file_path != null);
+ const file_path = optional_file_path orelse return;
+ _ = try self.addFile(file_path, null);
+ }
+
+ pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
+ self.hash.add(list_of_files.len);
+ for (list_of_files) |file_path| {
+ _ = try self.addFile(file_path, null);
+ }
+ }
+
+ /// Check the cache to see if the input exists in it. If it exists, returns `true`.
+ /// A hex encoding of its hash is available by calling `final`.
+ ///
+ /// This function will also acquire an exclusive lock to the manifest file. This means
+ /// that a process holding a Manifest will block any other process attempting to
+ /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
+ /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
+ /// file to be locked in exclusive mode.
+ ///
+ /// The lock on the manifest file is released when `deinit` is called. As another
+ /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
+ /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
+ pub fn hit(self: *Manifest) !bool {
+ const gpa = self.cache.gpa;
+ assert(self.manifest_file == null);
+
+ self.failed_file_index = null;
+
+ const ext = ".txt";
+ var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
+
+ var bin_digest: BinDigest = undefined;
+ self.hash.hasher.final(&bin_digest);
+
+ _ = std.fmt.bufPrint(
+ &self.hex_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&bin_digest)},
+ ) catch unreachable;
+
+ self.hash.hasher = hasher_init;
+ self.hash.hasher.update(&bin_digest);
+
+ mem.copy(u8, &manifest_file_path, &self.hex_digest);
+ manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
+
+ if (self.files.items.len == 0) {
+ // If there are no file inputs, we check if the manifest file exists instead of
+ // comparing the hashes on the files used for the cached item
+ while (true) {
+ if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .mode = .read_write,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.have_exclusive_lock = true;
+ break;
+ } else |open_err| switch (open_err) {
+ error.WouldBlock => {
+ self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .lock = .Shared,
+ });
+ break;
+ },
+ error.FileNotFound => {
+ if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
+ .read = true,
+ .truncate = false,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.manifest_dirty = true;
+ self.have_exclusive_lock = true;
+ return false; // cache miss; exclusive lock already held
+ } else |err| switch (err) {
+ error.WouldBlock => continue,
+ else => |e| return e,
+ }
+ },
+ else => |e| return e,
+ }
+ }
+ } else {
+ if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
+ .read = true,
+ .truncate = false,
+ .lock = .Exclusive,
+ .lock_nonblocking = self.want_shared_lock,
+ })) |manifest_file| {
+ self.manifest_file = manifest_file;
+ self.have_exclusive_lock = true;
+ } else |err| switch (err) {
+ error.WouldBlock => {
+ self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
+ .lock = .Shared,
+ });
+ },
+ else => |e| return e,
+ }
+ }
+
+ self.want_refresh_timestamp = true;
+
+ const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
+ defer gpa.free(file_contents);
+
+ const input_file_count = self.files.items.len;
+ var any_file_changed = false;
+ var line_iter = mem.tokenize(u8, file_contents, "\n");
+ var idx: usize = 0;
+ while (line_iter.next()) |line| {
+ defer idx += 1;
+
+ const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
+ const new = try self.files.addOne(gpa);
+ new.* = .{
+ .prefixed_path = null,
+ .contents = null,
+ .max_file_size = null,
+ .stat = undefined,
+ .bin_digest = undefined,
+ };
+ break :blk new;
+ };
+
+ var iter = mem.tokenize(u8, line, " ");
+ const size = iter.next() orelse return error.InvalidFormat;
+ const inode = iter.next() orelse return error.InvalidFormat;
+ const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
+ const digest_str = iter.next() orelse return error.InvalidFormat;
+ const prefix_str = iter.next() orelse return error.InvalidFormat;
+ const file_path = iter.rest();
+
+ cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
+ cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
+ cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
+ _ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
+ const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
+ if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
+
+ if (file_path.len == 0) {
+ return error.InvalidFormat;
+ }
+ if (cache_hash_file.prefixed_path) |pp| {
+ if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
+ return error.InvalidFormat;
+ }
+ }
+
+ if (cache_hash_file.prefixed_path == null) {
+ cache_hash_file.prefixed_path = .{
+ .prefix = prefix,
+ .sub_path = try gpa.dupe(u8, file_path),
+ };
+ }
+
+ const pp = cache_hash_file.prefixed_path.?;
+ const dir = self.cache.prefixes()[pp.prefix].handle;
+ const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
+ error.FileNotFound => {
+ try self.upgradeToExclusiveLock();
+ return false;
+ },
+ else => return error.CacheUnavailable,
+ };
+ defer this_file.close();
+
+ const actual_stat = this_file.stat() catch |err| {
+ self.failed_file_index = idx;
+ return err;
+ };
+ const size_match = actual_stat.size == cache_hash_file.stat.size;
+ const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
+ const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
+
+ if (!size_match or !mtime_match or !inode_match) {
+ self.manifest_dirty = true;
+
+ cache_hash_file.stat = .{
+ .size = actual_stat.size,
+ .mtime = actual_stat.mtime,
+ .inode = actual_stat.inode,
+ };
+
+ if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
+ // The actual file has an unreliable timestamp, force it to be hashed
+ cache_hash_file.stat.mtime = 0;
+ cache_hash_file.stat.inode = 0;
+ }
+
+ var actual_digest: BinDigest = undefined;
+ hashFile(this_file, &actual_digest) catch |err| {
+ self.failed_file_index = idx;
+ return err;
+ };
+
+ if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
+ cache_hash_file.bin_digest = actual_digest;
+ // keep going until we have the input file digests
+ any_file_changed = true;
+ }
+ }
+
+ if (!any_file_changed) {
+ self.hash.hasher.update(&cache_hash_file.bin_digest);
+ }
+ }
+
+ if (any_file_changed) {
+ // cache miss
+ // keep the manifest file open
+ self.unhit(bin_digest, input_file_count);
+ try self.upgradeToExclusiveLock();
+ return false;
+ }
+
+ if (idx < input_file_count) {
+ self.manifest_dirty = true;
+ while (idx < input_file_count) : (idx += 1) {
+ const ch_file = &self.files.items[idx];
+ self.populateFileHash(ch_file) catch |err| {
+ self.failed_file_index = idx;
+ return err;
+ };
+ }
+ try self.upgradeToExclusiveLock();
+ return false;
+ }
+
+ if (self.want_shared_lock) {
+ try self.downgradeToSharedLock();
+ }
+
+ return true;
+ }
+
+ pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
+ // Reset the hash.
+ self.hash.hasher = hasher_init;
+ self.hash.hasher.update(&bin_digest);
+
+ // Remove files not in the initial hash.
+ for (self.files.items[input_file_count..]) |*file| {
+ file.deinit(self.cache.gpa);
+ }
+ self.files.shrinkRetainingCapacity(input_file_count);
+
+ for (self.files.items) |file| {
+ self.hash.hasher.update(&file.bin_digest);
+ }
+ }
+
+ fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
+ // If the file_time is prior to the most recent problematic timestamp
+ // then we don't need to access the filesystem.
+ if (file_time < man.recent_problematic_timestamp)
+ return false;
+
+ // Next we will check the globally shared Cache timestamp, which is accessed
+ // from multiple threads.
+ man.cache.mutex.lock();
+ defer man.cache.mutex.unlock();
+
+ // Save the global one to our local one to avoid locking next time.
+ man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
+ if (file_time < man.recent_problematic_timestamp)
+ return false;
+
+ // This flag prevents multiple filesystem writes for the same hit() call.
+ if (man.want_refresh_timestamp) {
+ man.want_refresh_timestamp = false;
+
+ var file = man.cache.manifest_dir.createFile("timestamp", .{
+ .read = true,
+ .truncate = true,
+ }) catch return true;
+ defer file.close();
+
+ // Save locally and also save globally (we still hold the global lock).
+ man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
+ man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
+ }
+
+ return file_time >= man.recent_problematic_timestamp;
+ }
+
+ fn populateFileHash(self: *Manifest, ch_file: *File) !void {
+ const pp = ch_file.prefixed_path.?;
+ const dir = self.cache.prefixes()[pp.prefix].handle;
+ const file = try dir.openFile(pp.sub_path, .{});
+ defer file.close();
+
+ const actual_stat = try file.stat();
+ ch_file.stat = .{
+ .size = actual_stat.size,
+ .mtime = actual_stat.mtime,
+ .inode = actual_stat.inode,
+ };
+
+ if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
+ // The actual file has an unreliable timestamp, force it to be hashed
+ ch_file.stat.mtime = 0;
+ ch_file.stat.inode = 0;
+ }
+
+ if (ch_file.max_file_size) |max_file_size| {
+ if (ch_file.stat.size > max_file_size) {
+ return error.FileTooBig;
+ }
+
+ const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
+ errdefer self.cache.gpa.free(contents);
+
+ // Hash while reading from disk, to keep the contents in the cpu cache while
+ // doing hashing.
+ var hasher = hasher_init;
+ var off: usize = 0;
+ while (true) {
+ // give me everything you've got, captain
+ const bytes_read = try file.read(contents[off..]);
+ if (bytes_read == 0) break;
+ hasher.update(contents[off..][0..bytes_read]);
+ off += bytes_read;
+ }
+ hasher.final(&ch_file.bin_digest);
+
+ ch_file.contents = contents;
+ } else {
+ try hashFile(file, &ch_file.bin_digest);
+ }
+
+ self.hash.hasher.update(&ch_file.bin_digest);
+ }
+
+ /// Add a file as a dependency of process being cached, after the initial hash has been
+ /// calculated. This is useful for processes that don't know all the files that
+ /// are depended on ahead of time. For example, a source file that can import other files
+ /// will need to be recompiled if the imported file is changed.
+ pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
+ assert(self.manifest_file != null);
+
+ const gpa = self.cache.gpa;
+ const prefixed_path = try self.cache.findPrefix(file_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ const new_ch_file = try self.files.addOne(gpa);
+ new_ch_file.* = .{
+ .prefixed_path = prefixed_path,
+ .max_file_size = max_file_size,
+ .stat = undefined,
+ .bin_digest = undefined,
+ .contents = null,
+ };
+ errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+ try self.populateFileHash(new_ch_file);
+
+ return new_ch_file.contents.?;
+ }
+
+ /// Add a file as a dependency of process being cached, after the initial hash has been
+ /// calculated. This is useful for processes that don't know the all the files that
+ /// are depended on ahead of time. For example, a source file that can import other files
+ /// will need to be recompiled if the imported file is changed.
+ pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
+ assert(self.manifest_file != null);
+
+ const gpa = self.cache.gpa;
+ const prefixed_path = try self.cache.findPrefix(file_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ const new_ch_file = try self.files.addOne(gpa);
+ new_ch_file.* = .{
+ .prefixed_path = prefixed_path,
+ .max_file_size = null,
+ .stat = undefined,
+ .bin_digest = undefined,
+ .contents = null,
+ };
+ errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+ try self.populateFileHash(new_ch_file);
+ }
+
+ /// Like `addFilePost` but when the file contents have already been loaded from disk.
+ /// On success, cache takes ownership of `resolved_path`.
+ pub fn addFilePostContents(
+ self: *Manifest,
+ resolved_path: []u8,
+ bytes: []const u8,
+ stat: File.Stat,
+ ) error{OutOfMemory}!void {
+ assert(self.manifest_file != null);
+ const gpa = self.cache.gpa;
+
+ const ch_file = try self.files.addOne(gpa);
+ errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
+
+ const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
+ errdefer gpa.free(prefixed_path.sub_path);
+
+ ch_file.* = .{
+ .prefixed_path = prefixed_path,
+ .max_file_size = null,
+ .stat = stat,
+ .bin_digest = undefined,
+ .contents = null,
+ };
+
+ if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
+ // The actual file has an unreliable timestamp, force it to be hashed
+ ch_file.stat.mtime = 0;
+ ch_file.stat.inode = 0;
+ }
+
+ {
+ var hasher = hasher_init;
+ hasher.update(bytes);
+ hasher.final(&ch_file.bin_digest);
+ }
+
+ self.hash.hasher.update(&ch_file.bin_digest);
+ }
+
+ pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
+ assert(self.manifest_file != null);
+
+ const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
+ defer self.cache.gpa.free(dep_file_contents);
+
+ var error_buf = std.ArrayList(u8).init(self.cache.gpa);
+ defer error_buf.deinit();
+
+ var it: DepTokenizer = .{ .bytes = dep_file_contents };
+
+ // Skip first token: target.
+ switch (it.next() orelse return) { // Empty dep file OK.
+ .target, .target_must_resolve, .prereq => {},
+ else => |err| {
+ try err.printError(error_buf.writer());
+ log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
+ return error.InvalidDepFile;
+ },
+ }
+ // Process 0+ preqreqs.
+ // Clang is invoked in single-source mode so we never get more targets.
+ while (true) {
+ switch (it.next() orelse return) {
+ .target, .target_must_resolve => return,
+ .prereq => |file_path| try self.addFilePost(file_path),
+ else => |err| {
+ try err.printError(error_buf.writer());
+ log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
+ return error.InvalidDepFile;
+ },
+ }
+ }
+ }
+
+ /// Returns a hex encoded hash of the inputs.
+ pub fn final(self: *Manifest) [hex_digest_len]u8 {
+ assert(self.manifest_file != null);
+
+ // We don't close the manifest file yet, because we want to
+ // keep it locked until the API user is done using it.
+ // We also don't write out the manifest yet, because until
+ // cache_release is called we still might be working on creating
+ // the artifacts to cache.
+
+ var bin_digest: BinDigest = undefined;
+ self.hash.hasher.final(&bin_digest);
+
+ var out_digest: [hex_digest_len]u8 = undefined;
+ _ = std.fmt.bufPrint(
+ &out_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&bin_digest)},
+ ) catch unreachable;
+
+ return out_digest;
+ }
+
+ /// If `want_shared_lock` is true, this function automatically downgrades the
+ /// lock from exclusive to shared.
+ pub fn writeManifest(self: *Manifest) !void {
+ assert(self.have_exclusive_lock);
+
+ const manifest_file = self.manifest_file.?;
+ if (self.manifest_dirty) {
+ self.manifest_dirty = false;
+
+ var contents = std.ArrayList(u8).init(self.cache.gpa);
+ defer contents.deinit();
+
+ const writer = contents.writer();
+ var encoded_digest: [hex_digest_len]u8 = undefined;
+
+ for (self.files.items) |file| {
+ _ = std.fmt.bufPrint(
+ &encoded_digest,
+ "{s}",
+ .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
+ ) catch unreachable;
+ try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
+ file.stat.size,
+ file.stat.inode,
+ file.stat.mtime,
+ &encoded_digest,
+ file.prefixed_path.?.prefix,
+ file.prefixed_path.?.sub_path,
+ });
+ }
+
+ try manifest_file.setEndPos(contents.items.len);
+ try manifest_file.pwriteAll(contents.items, 0);
+ }
+
+ if (self.want_shared_lock) {
+ try self.downgradeToSharedLock();
+ }
+ }
+
+ fn downgradeToSharedLock(self: *Manifest) !void {
+ if (!self.have_exclusive_lock) return;
+
+ // WASI does not currently support flock, so we bypass it here.
+ // TODO: If/when flock is supported on WASI, this check should be removed.
+ // See https://github.com/WebAssembly/wasi-filesystem/issues/2
+ if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
+ const manifest_file = self.manifest_file.?;
+ try manifest_file.downgradeLock();
+ }
+
+ self.have_exclusive_lock = false;
+ }
+
+ fn upgradeToExclusiveLock(self: *Manifest) !void {
+ if (self.have_exclusive_lock) return;
+ assert(self.manifest_file != null);
+
+ // WASI does not currently support flock, so we bypass it here.
+ // TODO: If/when flock is supported on WASI, this check should be removed.
+ // See https://github.com/WebAssembly/wasi-filesystem/issues/2
+ if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
+ const manifest_file = self.manifest_file.?;
+ // Here we intentionally have a period where the lock is released, in case there are
+ // other processes holding a shared lock.
+ manifest_file.unlock();
+ try manifest_file.lock(.Exclusive);
+ }
+ self.have_exclusive_lock = true;
+ }
+
+ /// Obtain only the data needed to maintain a lock on the manifest file.
+ /// The `Manifest` remains safe to deinit.
+ /// Don't forget to call `writeManifest` before this!
+ pub fn toOwnedLock(self: *Manifest) Lock {
+ const lock: Lock = .{
+ .manifest_file = self.manifest_file.?,
+ };
+
+ self.manifest_file = null;
+ return lock;
+ }
+
+ /// Releases the manifest file and frees any memory the Manifest was using.
+ /// `Manifest.hit` must be called first.
+ /// Don't forget to call `writeManifest` before this!
+ pub fn deinit(self: *Manifest) void {
+ if (self.manifest_file) |file| {
+ if (builtin.os.tag == .windows) {
+ // See Lock.release for why this is required on Windows
+ file.unlock();
+ }
+
+ file.close();
+ }
+ for (self.files.items) |*file| {
+ file.deinit(self.cache.gpa);
+ }
+ self.files.deinit(self.cache.gpa);
+ }
+};
+
+/// On operating systems that support symlinks, does a readlink. On other operating systems,
+/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
+/// it is treated as not supporting symlinks.
+pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
+ if (builtin.os.tag == .windows) {
+ return dir.readFile(sub_path, buffer);
+ } else {
+ return dir.readLink(sub_path, buffer);
+ }
+}
+
+/// On operating systems that support symlinks, does a symlink. On other operating systems,
+/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
+/// it is treated as not supporting symlinks.
+/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
+pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
+ assert(data.len <= 255);
+ if (builtin.os.tag == .windows) {
+ return dir.writeFile(sub_path, data);
+ } else {
+ return dir.symLink(data, sub_path, .{});
+ }
+}
+
+fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
+ var buf: [1024]u8 = undefined;
+
+ var hasher = hasher_init;
+ while (true) {
+ const bytes_read = try file.read(&buf);
+ if (bytes_read == 0) break;
+ hasher.update(buf[0..bytes_read]);
+ }
+
+ hasher.final(bin_digest);
+}
+
+// Create/Write a file, close it, then grab its stat.mtime timestamp.
+fn testGetCurrentFileTimestamp() !i128 {
+ var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
+ .read = true,
+ .truncate = true,
+ });
+ defer file.close();
+
+ return (try file.stat()).mtime;
+}
+
+test "cache file and then recall it" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+
+ const cwd = fs.cwd();
+
+ const temp_file = "test.txt";
+ const temp_manifest_dir = "temp_manifest_dir";
+
+ try cwd.writeFile(temp_file, "Hello, world!\n");
+
+ // Wait for file timestamps to tick
+ const initial_time = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time) {
+ std.time.sleep(1);
+ }
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+
+ {
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.add(true);
+ ch.hash.add(@as(u16, 1234));
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file, null);
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try ch.hit());
+
+ digest1 = ch.final();
+ try ch.writeManifest();
+ }
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.add(true);
+ ch.hash.add(@as(u16, 1234));
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file, null);
+
+ // Cache hit! We just "built" the same file
+ try testing.expect(try ch.hit());
+ digest2 = ch.final();
+
+ try testing.expectEqual(false, ch.have_exclusive_lock);
+ }
+
+ try testing.expectEqual(digest1, digest2);
+ }
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteFile(temp_file);
+}
+
+test "check that changing a file makes cache fail" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+ const cwd = fs.cwd();
+
+ const temp_file = "cache_hash_change_file_test.txt";
+ const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
+ const original_temp_file_contents = "Hello, world!\n";
+ const updated_temp_file_contents = "Hello, world; but updated!\n";
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteTree(temp_file);
+
+ try cwd.writeFile(temp_file, original_temp_file_contents);
+
+ // Wait for file timestamps to tick
+ const initial_time = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time) {
+ std.time.sleep(1);
+ }
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+
+ {
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ const temp_file_idx = try ch.addFile(temp_file, 100);
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try ch.hit());
+
+ try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
+
+ digest1 = ch.final();
+
+ try ch.writeManifest();
+ }
+
+ try cwd.writeFile(temp_file, updated_temp_file_contents);
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ const temp_file_idx = try ch.addFile(temp_file, 100);
+
+ // A file that we depend on has been updated, so the cache should not contain an entry for it
+ try testing.expectEqual(false, try ch.hit());
+
+ // The cache system does not keep the contents of re-hashed input files.
+ try testing.expect(ch.files.items[temp_file_idx].contents == null);
+
+ digest2 = ch.final();
+
+ try ch.writeManifest();
+ }
+
+ try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
+ }
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteTree(temp_file);
+}
+
+test "no file inputs" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+ const cwd = fs.cwd();
+ const temp_manifest_dir = "no_file_inputs_manifest_dir";
+ defer cwd.deleteTree(temp_manifest_dir) catch {};
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var man = cache.obtain();
+ defer man.deinit();
+
+ man.hash.addBytes("1234");
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try man.hit());
+
+ digest1 = man.final();
+
+ try man.writeManifest();
+ }
+ {
+ var man = cache.obtain();
+ defer man.deinit();
+
+ man.hash.addBytes("1234");
+
+ try testing.expect(try man.hit());
+ digest2 = man.final();
+ try testing.expectEqual(false, man.have_exclusive_lock);
+ }
+
+ try testing.expectEqual(digest1, digest2);
+}
+
+test "Manifest with files added after initial hash work" {
+ if (builtin.os.tag == .wasi) {
+ // https://github.com/ziglang/zig/issues/5437
+ return error.SkipZigTest;
+ }
+ const cwd = fs.cwd();
+
+ const temp_file1 = "cache_hash_post_file_test1.txt";
+ const temp_file2 = "cache_hash_post_file_test2.txt";
+ const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
+
+ try cwd.writeFile(temp_file1, "Hello, world!\n");
+ try cwd.writeFile(temp_file2, "Hello world the second!\n");
+
+ // Wait for file timestamps to tick
+ const initial_time = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time) {
+ std.time.sleep(1);
+ }
+
+ var digest1: [hex_digest_len]u8 = undefined;
+ var digest2: [hex_digest_len]u8 = undefined;
+ var digest3: [hex_digest_len]u8 = undefined;
+
+ {
+ var cache = Cache{
+ .gpa = testing.allocator,
+ .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
+ };
+ cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
+ defer cache.manifest_dir.close();
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file1, null);
+
+ // There should be nothing in the cache
+ try testing.expectEqual(false, try ch.hit());
+
+ _ = try ch.addFilePost(temp_file2);
+
+ digest1 = ch.final();
+ try ch.writeManifest();
+ }
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file1, null);
+
+ try testing.expect(try ch.hit());
+ digest2 = ch.final();
+
+ try testing.expectEqual(false, ch.have_exclusive_lock);
+ }
+ try testing.expect(mem.eql(u8, &digest1, &digest2));
+
+ // Modify the file added after initial hash
+ try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
+
+ // Wait for file timestamps to tick
+ const initial_time2 = try testGetCurrentFileTimestamp();
+ while ((try testGetCurrentFileTimestamp()) == initial_time2) {
+ std.time.sleep(1);
+ }
+
+ {
+ var ch = cache.obtain();
+ defer ch.deinit();
+
+ ch.hash.addBytes("1234");
+ _ = try ch.addFile(temp_file1, null);
+
+ // A file that we depend on has been updated, so the cache should not contain an entry for it
+ try testing.expectEqual(false, try ch.hit());
+
+ _ = try ch.addFilePost(temp_file2);
+
+ digest3 = ch.final();
+
+ try ch.writeManifest();
+ }
+
+ try testing.expect(!mem.eql(u8, &digest1, &digest3));
+ }
+
+ try cwd.deleteTree(temp_manifest_dir);
+ try cwd.deleteFile(temp_file1);
+ try cwd.deleteFile(temp_file2);
+}
diff --git a/src/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig
diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig
@@ -83,7 +83,7 @@ max_memory: ?u64 = null,
shared_memory: bool = false,
global_base: ?u64 = null,
c_std: std.Build.CStd,
-override_lib_dir: ?[]const u8,
+zig_lib_dir: ?[]const u8,
main_pkg_path: ?[]const u8,
exec_cmd_args: ?[]const ?[]const u8,
name_prefix: []const u8,
@@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {
.installed_headers = ArrayList(*Step).init(builder.allocator),
.object_src = undefined,
.c_std = std.Build.CStd.C99,
- .override_lib_dir = null,
+ .zig_lib_dir = null,
.main_pkg_path = null,
.exec_cmd_args = null,
.name_prefix = "",
@@ -506,26 +506,11 @@ pub fn installLibraryHeaders(a: *CompileStep, l: *CompileStep) void {
a.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM");
}
-/// Creates a `RunStep` with an executable built with `addExecutable`.
-/// Add command line arguments with `addArg`.
+/// Deprecated: use `std.Build.addRunArtifact`
+/// This function will run in the context of the package that created the executable,
+/// which is undesirable when running an executable provided by a dependency package.
pub fn run(exe: *CompileStep) *RunStep {
- assert(exe.kind == .exe or exe.kind == .test_exe);
-
- // It doesn't have to be native. We catch that if you actually try to run it.
- // Consider that this is declarative; the run step may not be run unless a user
- // option is supplied.
- const run_step = RunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name}));
- run_step.addArtifactArg(exe);
-
- if (exe.kind == .test_exe) {
- run_step.addArg(exe.builder.zig_exe);
- }
-
- if (exe.vcpkg_bin_path) |path| {
- run_step.addPathDir(path);
- }
-
- return run_step;
+ return exe.builder.addRunArtifact(exe);
}
/// Creates an `EmulatableRunStep` with an executable built with `addExecutable`.
@@ -872,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {
}
pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
- self.override_lib_dir = self.builder.dupePath(dir_path);
+ self.zig_lib_dir = self.builder.dupePath(dir_path);
}
pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
@@ -1365,10 +1350,10 @@ fn make(step: *Step) !void {
}
try zig_args.append("--cache-dir");
- try zig_args.append(builder.pathFromRoot(builder.cache_root));
+ try zig_args.append(builder.cache_root.path orelse ".");
try zig_args.append("--global-cache-dir");
- try zig_args.append(builder.pathFromRoot(builder.global_cache_root));
+ try zig_args.append(builder.global_cache_root.path orelse ".");
try zig_args.append("--name");
try zig_args.append(self.name);
@@ -1718,12 +1703,12 @@ fn make(step: *Step) !void {
try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
try addFlag(&zig_args, "build-id", self.build_id);
- if (self.override_lib_dir) |dir| {
+ if (self.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
try zig_args.append(builder.pathFromRoot(dir));
- } else if (builder.override_lib_dir) |dir| {
+ } else if (builder.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
- try zig_args.append(builder.pathFromRoot(dir));
+ try zig_args.append(dir);
}
if (self.main_pkg_path) |dir| {
@@ -1760,23 +1745,15 @@ fn make(step: *Step) !void {
args_length += arg.len + 1; // +1 to account for null terminator
}
if (args_length >= 30 * 1024) {
- const args_dir = try fs.path.join(
- builder.allocator,
- &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
- );
- try std.fs.cwd().makePath(args_dir);
-
- var args_arena = std.heap.ArenaAllocator.init(builder.allocator);
- defer args_arena.deinit();
+ try builder.cache_root.handle.makePath("args");
const args_to_escape = zig_args.items[2..];
- var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len);
-
+ var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len);
arg_blk: for (args_to_escape) |arg| {
for (arg) |c, arg_idx| {
if (c == '\\' or c == '"') {
// Slow path for arguments that need to be escaped. We'll need to allocate and copy
- var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1);
+ var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1);
const writer = escaped.writer();
try writer.writeAll(arg[0..arg_idx]);
for (arg[arg_idx..]) |to_escape| {
@@ -1804,11 +1781,16 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&args_hash)},
);
- const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });
- try std.fs.cwd().writeFile(args_file, args);
+ const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
+ try builder.cache_root.handle.writeFile(args_file, args);
+
+ const resolved_args_file = try mem.concat(builder.allocator, u8, &.{
+ "@",
+ try builder.cache_root.join(builder.allocator, &.{args_file}),
+ });
zig_args.shrinkRetainingCapacity(2);
- try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));
+ try zig_args.append(resolved_args_file);
}
const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);
diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig
@@ -13,11 +13,13 @@ pub const Style = union(enum) {
cmake: std.Build.FileSource,
/// Instead of starting with an input file, start with nothing.
blank,
+ /// Start with nothing, like blank, and output a nasm .asm file.
+ nasm,
pub fn getFileSource(style: Style) ?std.Build.FileSource {
switch (style) {
.autoconf, .cmake => |s| return s,
- .blank => return null,
+ .blank, .nasm => return null,
}
}
};
@@ -84,6 +86,10 @@ pub fn addValues(self: *ConfigHeaderStep, values: anytype) void {
return addValuesInner(self, values) catch @panic("OOM");
}
+pub fn getFileSource(self: *ConfigHeaderStep) std.Build.FileSource {
+ return .{ .generated = &self.output_file };
+}
+
fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void {
inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| {
try putValue(self, field.name, field.type, @field(values, field.name));
@@ -125,6 +131,12 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v
return;
}
},
+ .Int => {
+ if (ptr.size == .Slice and ptr.child == u8) {
+ try self.values.put(field_name, .{ .string = v });
+ return;
+ }
+ },
else => {},
}
@@ -158,22 +170,31 @@ fn make(step: *Step) !void {
var output = std.ArrayList(u8).init(gpa);
defer output.deinit();
- try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n");
+ const header_text = "This file was generated by ConfigHeaderStep using the Zig Build System.";
+ const c_generated_line = "/* " ++ header_text ++ " */\n";
+ const asm_generated_line = "; " ++ header_text ++ "\n";
switch (self.style) {
.autoconf => |file_source| {
+ try output.appendSlice(c_generated_line);
const src_path = file_source.getPath(self.builder);
const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
try render_autoconf(contents, &output, self.values, src_path);
},
.cmake => |file_source| {
+ try output.appendSlice(c_generated_line);
const src_path = file_source.getPath(self.builder);
const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes);
try render_cmake(contents, &output, self.values, src_path);
},
.blank => {
+ try output.appendSlice(c_generated_line);
try render_blank(&output, self.values, self.include_path);
},
+ .nasm => {
+ try output.appendSlice(asm_generated_line);
+ try render_nasm(&output, self.values);
+ },
}
hash.update(output.items);
@@ -187,9 +208,7 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
- const output_dir = try std.fs.path.join(gpa, &[_][]const u8{
- self.builder.cache_root, "o", &hash_basename,
- });
+ const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename });
// If output_path has directory parts, deal with them. Example:
// output_dir is zig-cache/o/HASH
@@ -247,7 +266,7 @@ fn render_autoconf(
any_errors = true;
continue;
};
- try renderValue(output, name, kv.value);
+ try renderValueC(output, name, kv.value);
}
for (values_copy.keys()) |name| {
@@ -298,7 +317,7 @@ fn render_cmake(
any_errors = true;
continue;
};
- try renderValue(output, name, kv.value);
+ try renderValueC(output, name, kv.value);
}
for (values_copy.keys()) |name| {
@@ -332,7 +351,7 @@ fn render_blank(
const values = defines.values();
for (defines.keys()) |name, i| {
- try renderValue(output, name, values[i]);
+ try renderValueC(output, name, values[i]);
}
try output.appendSlice("#endif /* ");
@@ -340,7 +359,14 @@ fn render_blank(
try output.appendSlice(" */\n");
}
-fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
+fn render_nasm(output: *std.ArrayList(u8), defines: std.StringArrayHashMap(Value)) !void {
+ const values = defines.values();
+ for (defines.keys()) |name, i| {
+ try renderValueNasm(output, name, values[i]);
+ }
+}
+
+fn renderValueC(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
switch (value) {
.undef => {
try output.appendSlice("/* #undef ");
@@ -370,3 +396,33 @@ fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void
},
}
}
+
+fn renderValueNasm(output: *std.ArrayList(u8), name: []const u8, value: Value) !void {
+ switch (value) {
+ .undef => {
+ try output.appendSlice("; %undef ");
+ try output.appendSlice(name);
+ try output.appendSlice("\n");
+ },
+ .defined => {
+ try output.appendSlice("%define ");
+ try output.appendSlice(name);
+ try output.appendSlice("\n");
+ },
+ .boolean => |b| {
+ try output.appendSlice("%define ");
+ try output.appendSlice(name);
+ try output.appendSlice(if (b) " 1\n" else " 0\n");
+ },
+ .int => |i| {
+ try output.writer().print("%define {s} {d}\n", .{ name, i });
+ },
+ .ident => |ident| {
+ try output.writer().print("%define {s} {s}\n", .{ name, ident });
+ },
+ .string => |string| {
+ // TODO: use nasm-specific escaping instead of zig string literals
+ try output.writer().print("%define {s} \"{}\"\n", .{ name, std.zig.fmtEscapes(string) });
+ },
+ }
+}
diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig
@@ -234,26 +234,20 @@ fn make(step: *Step) !void {
);
}
- const options_directory = self.builder.pathFromRoot(
- try fs.path.join(
- self.builder.allocator,
- &[_][]const u8{ self.builder.cache_root, "options" },
- ),
- );
-
- try fs.cwd().makePath(options_directory);
+ var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{});
+ defer options_dir.close();
- const options_file = try fs.path.join(
- self.builder.allocator,
- &[_][]const u8{ options_directory, &self.hashContentsToFileName() },
- );
+ const basename = self.hashContentsToFileName();
- try fs.cwd().writeFile(options_file, self.contents.items);
+ try options_dir.writeFile(&basename, self.contents.items);
- self.generated_file.path = options_file;
+ self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{
+ "options", &basename,
+ });
}
fn hashContentsToFileName(self: *OptionsStep) [64]u8 {
+ // TODO update to use the cache system instead of this
// This implementation is copied from `WriteFileStep.make`
var hash = std.crypto.hash.blake2.Blake2b384.init(.{});
@@ -289,13 +283,19 @@ test "OptionsStep" {
const host = try std.zig.system.NativeTargetInfo.detect(.{});
+ var cache: std.Build.Cache = .{
+ .gpa = arena.allocator(),
+ .manifest_dir = std.fs.cwd(),
+ };
+
var builder = try std.Build.create(
arena.allocator(),
"test",
- "test",
- "test",
- "test",
+ .{ .path = "test", .handle = std.fs.cwd() },
+ .{ .path = "test", .handle = std.fs.cwd() },
+ .{ .path = "test", .handle = std.fs.cwd() },
host,
+ &cache,
);
defer builder.destroy();
diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig
@@ -39,6 +39,14 @@ expected_exit_code: ?u8 = 0,
/// Print the command before running it
print: bool,
+/// Controls whether execution is skipped if the output file is up-to-date.
+/// The default is to always run if there is no output file, and to skip
+/// running if all output files are up-to-date.
+condition: enum { output_outdated, always } = .output_outdated,
+
+/// Additional file paths relative to build.zig that, when modified, indicate
+/// that the RunStep should be re-executed.
+extra_file_dependencies: []const []const u8 = &.{},
pub const StdIoAction = union(enum) {
inherit,
@@ -51,6 +59,12 @@ pub const Arg = union(enum) {
artifact: *CompileStep,
file_source: std.Build.FileSource,
bytes: []u8,
+ output: Output,
+
+ pub const Output = struct {
+ generated_file: *std.Build.GeneratedFile,
+ basename: []const u8,
+ };
};
pub fn create(builder: *std.Build, name: []const u8) *RunStep {
@@ -71,6 +85,20 @@ pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void {
self.step.dependOn(&artifact.step);
}
+/// This provides file path as a command line argument to the command being
+/// run, and returns a FileSource which can be used as inputs to other APIs
+/// throughout the build system.
+pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource {
+ const generated_file = rs.builder.allocator.create(std.Build.GeneratedFile) catch @panic("OOM");
+ generated_file.* = .{ .step = &rs.step };
+ rs.argv.append(.{ .output = .{
+ .generated_file = generated_file,
+ .basename = rs.builder.dupe(basename),
+ } }) catch @panic("OOM");
+
+ return .{ .generated = generated_file };
+}
+
pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void {
self.argv.append(Arg{
.file_source = file_source.dupe(self.builder),
@@ -159,25 +187,105 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
};
}
+fn needOutputCheck(self: RunStep) bool {
+ if (self.extra_file_dependencies.len > 0) return true;
+
+ for (self.argv.items) |arg| switch (arg) {
+ .output => return true,
+ else => continue,
+ };
+
+ return switch (self.condition) {
+ .always => false,
+ .output_outdated => true,
+ };
+}
+
fn make(step: *Step) !void {
const self = @fieldParentPtr(RunStep, "step", step);
+ const need_output_check = self.needOutputCheck();
var argv_list = ArrayList([]const u8).init(self.builder.allocator);
+ var output_placeholders = ArrayList(struct {
+ index: usize,
+ output: Arg.Output,
+ }).init(self.builder.allocator);
+
+ var man = self.builder.cache.obtain();
+ defer man.deinit();
+
for (self.argv.items) |arg| {
switch (arg) {
- .bytes => |bytes| try argv_list.append(bytes),
- .file_source => |file| try argv_list.append(file.getPath(self.builder)),
+ .bytes => |bytes| {
+ try argv_list.append(bytes);
+ man.hash.addBytes(bytes);
+ },
+ .file_source => |file| {
+ const file_path = file.getPath(self.builder);
+ try argv_list.append(file_path);
+ _ = try man.addFile(file_path, null);
+ },
.artifact => |artifact| {
if (artifact.target.isWindows()) {
// On Windows we don't have rpaths so we have to add .dll search paths to PATH
self.addPathForDynLibs(artifact);
}
- const executable_path = artifact.installed_path orelse artifact.getOutputSource().getPath(self.builder);
- try argv_list.append(executable_path);
+ const file_path = artifact.installed_path orelse
+ artifact.getOutputSource().getPath(self.builder);
+
+ try argv_list.append(file_path);
+
+ _ = try man.addFile(file_path, null);
+ },
+ .output => |output| {
+ man.hash.addBytes(output.basename);
+ // Add a placeholder into the argument list because we need the
+ // manifest hash to be updated with all arguments before the
+ // object directory is computed.
+ try argv_list.append("");
+ try output_placeholders.append(.{
+ .index = argv_list.items.len - 1,
+ .output = output,
+ });
},
}
}
+ if (need_output_check) {
+ for (self.extra_file_dependencies) |file_path| {
+ _ = try man.addFile(self.builder.pathFromRoot(file_path), null);
+ }
+
+ if (man.hit() catch |err| failWithCacheError(man, err)) {
+ // cache hit, skip running command
+ const digest = man.final();
+ for (output_placeholders.items) |placeholder| {
+ placeholder.output.generated_file.path = try self.builder.cache_root.join(
+ self.builder.allocator,
+ &.{ "o", &digest, placeholder.output.basename },
+ );
+ }
+ return;
+ }
+
+ const digest = man.final();
+
+ for (output_placeholders.items) |placeholder| {
+ const output_path = try self.builder.cache_root.join(
+ self.builder.allocator,
+ &.{ "o", &digest, placeholder.output.basename },
+ );
+ const output_dir = fs.path.dirname(output_path).?;
+ fs.cwd().makePath(output_dir) catch |err| {
+ std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
+ return err;
+ };
+
+ placeholder.output.generated_file.path = output_path;
+ argv_list.items[placeholder.index] = output_path;
+ }
+ }
+
try runCommand(
argv_list.items,
self.builder,
@@ -189,6 +297,10 @@ fn make(step: *Step) !void {
self.cwd,
self.print,
);
+
+ if (need_output_check) {
+ try man.writeManifest();
+ }
}
pub fn runCommand(
@@ -202,11 +314,13 @@ pub fn runCommand(
maybe_cwd: ?[]const u8,
print: bool,
) !void {
- const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root;
+ const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path;
if (!std.process.can_spawn) {
const cmd = try std.mem.join(builder.allocator, " ", argv);
- std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });
+ std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{
+ @tagName(builtin.os.tag), cmd,
+ });
builder.allocator.free(cmd);
return ExecError.ExecNotSupported;
}
@@ -347,6 +461,19 @@ pub fn runCommand(
}
}
+fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
+ const i = man.failed_file_index orelse failWithSimpleError(err);
+ const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
+ const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
+ std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
+ std.process.exit(1);
+}
+
+fn failWithSimpleError(err: anyerror) noreturn {
+ std.debug.print("{s}\n", .{@errorName(err)});
+ std.process.exit(1);
+}
+
fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd});
for (argv) |arg| {
diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig
@@ -85,8 +85,8 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
- const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
- self.builder.cache_root, "o", &hash_basename,
+ const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
+ "o", &hash_basename,
});
var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
diff --git a/src/Cache.zig b/src/Cache.zig
@@ -1,1265 +0,0 @@
-//! Manages `zig-cache` directories.
-//! This is not a general-purpose cache. It is designed to be fast and simple,
-//! not to withstand attacks using specially-crafted input.
-
-gpa: Allocator,
-manifest_dir: fs.Dir,
-hash: HashHelper = .{},
-/// This value is accessed from multiple threads, protected by mutex.
-recent_problematic_timestamp: i128 = 0,
-mutex: std.Thread.Mutex = .{},
-
-/// A set of strings such as the zig library directory or project source root, which
-/// are stripped from the file paths before putting into the cache. They
-/// are replaced with single-character indicators. This is not to save
-/// space but to eliminate absolute file paths. This improves portability
-/// and usefulness of the cache for advanced use cases.
-prefixes_buffer: [3]Compilation.Directory = undefined,
-prefixes_len: usize = 0,
-
-const Cache = @This();
-const std = @import("std");
-const builtin = @import("builtin");
-const crypto = std.crypto;
-const fs = std.fs;
-const assert = std.debug.assert;
-const testing = std.testing;
-const mem = std.mem;
-const fmt = std.fmt;
-const Allocator = std.mem.Allocator;
-const Compilation = @import("Compilation.zig");
-const log = std.log.scoped(.cache);
-
-pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void {
- if (directory.path) |p| {
- log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p });
- }
- cache.prefixes_buffer[cache.prefixes_len] = directory;
- cache.prefixes_len += 1;
-}
-
-/// Be sure to call `Manifest.deinit` after successful initialization.
-pub fn obtain(cache: *Cache) Manifest {
- return Manifest{
- .cache = cache,
- .hash = cache.hash,
- .manifest_file = null,
- .manifest_dirty = false,
- .hex_digest = undefined,
- };
-}
-
-pub fn prefixes(cache: *const Cache) []const Compilation.Directory {
- return cache.prefixes_buffer[0..cache.prefixes_len];
-}
-
-const PrefixedPath = struct {
- prefix: u8,
- sub_path: []u8,
-};
-
-fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
- const gpa = cache.gpa;
- const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
- errdefer gpa.free(resolved_path);
- return findPrefixResolved(cache, resolved_path);
-}
-
-/// Takes ownership of `resolved_path` on success.
-fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
- const gpa = cache.gpa;
- const prefixes_slice = cache.prefixes();
- var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
- while (i < prefixes_slice.len) : (i += 1) {
- const p = prefixes_slice[i].path.?;
- if (mem.startsWith(u8, resolved_path, p)) {
- // +1 to skip over the path separator here
- const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
- gpa.free(resolved_path);
- return PrefixedPath{
- .prefix = @intCast(u8, i),
- .sub_path = sub_path,
- };
- } else {
- log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p });
- }
- }
-
- return PrefixedPath{
- .prefix = 0,
- .sub_path = resolved_path,
- };
-}
-
-/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
-pub const bin_digest_len = 16;
-pub const hex_digest_len = bin_digest_len * 2;
-pub const BinDigest = [bin_digest_len]u8;
-
-const manifest_file_size_max = 50 * 1024 * 1024;
-
-/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
-/// provides enough collision resistance for the Manifest use cases, while being one of our
-/// fastest options right now.
-pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
-
-/// Initial state, that can be copied.
-pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
-
-pub const File = struct {
- prefixed_path: ?PrefixedPath,
- max_file_size: ?usize,
- stat: Stat,
- bin_digest: BinDigest,
- contents: ?[]const u8,
-
- pub const Stat = struct {
- inode: fs.File.INode,
- size: u64,
- mtime: i128,
- };
-
- pub fn deinit(self: *File, gpa: Allocator) void {
- if (self.prefixed_path) |pp| {
- gpa.free(pp.sub_path);
- self.prefixed_path = null;
- }
- if (self.contents) |contents| {
- gpa.free(contents);
- self.contents = null;
- }
- self.* = undefined;
- }
-};
-
-pub const HashHelper = struct {
- hasher: Hasher = hasher_init,
-
- const EmitLoc = Compilation.EmitLoc;
-
- /// Record a slice of bytes as an dependency of the process being cached
- pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
- hh.hasher.update(mem.asBytes(&bytes.len));
- hh.hasher.update(bytes);
- }
-
- pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
- hh.add(optional_bytes != null);
- hh.addBytes(optional_bytes orelse return);
- }
-
- pub fn addEmitLoc(hh: *HashHelper, emit_loc: EmitLoc) void {
- hh.addBytes(emit_loc.basename);
- }
-
- pub fn addOptionalEmitLoc(hh: *HashHelper, optional_emit_loc: ?EmitLoc) void {
- hh.add(optional_emit_loc != null);
- hh.addEmitLoc(optional_emit_loc orelse return);
- }
-
- pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
- hh.add(list_of_bytes.len);
- for (list_of_bytes) |bytes| hh.addBytes(bytes);
- }
-
- /// Convert the input value into bytes and record it as a dependency of the process being cached.
- pub fn add(hh: *HashHelper, x: anytype) void {
- switch (@TypeOf(x)) {
- std.builtin.Version => {
- hh.add(x.major);
- hh.add(x.minor);
- hh.add(x.patch);
- },
- std.Target.Os.TaggedVersionRange => {
- switch (x) {
- .linux => |linux| {
- hh.add(linux.range.min);
- hh.add(linux.range.max);
- hh.add(linux.glibc);
- },
- .windows => |windows| {
- hh.add(windows.min);
- hh.add(windows.max);
- },
- .semver => |semver| {
- hh.add(semver.min);
- hh.add(semver.max);
- },
- .none => {},
- }
- },
- else => switch (@typeInfo(@TypeOf(x))) {
- .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
- else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
- },
- }
- }
-
- pub fn addOptional(hh: *HashHelper, optional: anytype) void {
- hh.add(optional != null);
- hh.add(optional orelse return);
- }
-
- /// Returns a hex encoded hash of the inputs, without modifying state.
- pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
- var copy = hh;
- return copy.final();
- }
-
- pub fn peekBin(hh: HashHelper) BinDigest {
- var copy = hh;
- var bin_digest: BinDigest = undefined;
- copy.hasher.final(&bin_digest);
- return bin_digest;
- }
-
- /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
- pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
- var bin_digest: BinDigest = undefined;
- hh.hasher.final(&bin_digest);
-
- var out_digest: [hex_digest_len]u8 = undefined;
- _ = std.fmt.bufPrint(
- &out_digest,
- "{s}",
- .{std.fmt.fmtSliceHexLower(&bin_digest)},
- ) catch unreachable;
- return out_digest;
- }
-};
-
-pub const Lock = struct {
- manifest_file: fs.File,
-
- pub fn release(lock: *Lock) void {
- if (builtin.os.tag == .windows) {
- // Windows does not guarantee that locks are immediately unlocked when
- // the file handle is closed. See LockFileEx documentation.
- lock.manifest_file.unlock();
- }
-
- lock.manifest_file.close();
- lock.* = undefined;
- }
-};
-
-pub const Manifest = struct {
- cache: *Cache,
- /// Current state for incremental hashing.
- hash: HashHelper,
- manifest_file: ?fs.File,
- manifest_dirty: bool,
- /// Set this flag to true before calling hit() in order to indicate that
- /// upon a cache hit, the code using the cache will not modify the files
- /// within the cache directory. This allows multiple processes to utilize
- /// the same cache directory at the same time.
- want_shared_lock: bool = true,
- have_exclusive_lock: bool = false,
- // Indicate that we want isProblematicTimestamp to perform a filesystem write in
- // order to obtain a problematic timestamp for the next call. Calls after that
- // will then use the same timestamp, to avoid unnecessary filesystem writes.
- want_refresh_timestamp: bool = true,
- files: std.ArrayListUnmanaged(File) = .{},
- hex_digest: [hex_digest_len]u8,
- /// Populated when hit() returns an error because of one
- /// of the files listed in the manifest.
- failed_file_index: ?usize = null,
- /// Keeps track of the last time we performed a file system write to observe
- /// what time the file system thinks it is, according to its own granularity.
- recent_problematic_timestamp: i128 = 0,
-
- /// Add a file as a dependency of process being cached. When `hit` is
- /// called, the file's contents will be checked to ensure that it matches
- /// the contents from previous times.
- ///
- /// Max file size will be used to determine the amount of space the file contents
- /// are allowed to take up in memory. If max_file_size is null, then the contents
- /// will not be loaded into memory.
- ///
- /// Returns the index of the entry in the `files` array list. You can use it
- /// to access the contents of the file after calling `hit()` like so:
- ///
- /// ```
- /// var file_contents = cache_hash.files.items[file_index].contents.?;
- /// ```
- pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
- assert(self.manifest_file == null);
-
- const gpa = self.cache.gpa;
- try self.files.ensureUnusedCapacity(gpa, 1);
- const prefixed_path = try self.cache.findPrefix(file_path);
- errdefer gpa.free(prefixed_path.sub_path);
-
- log.debug("Manifest.addFile {s} -> {d} {s}", .{
- file_path, prefixed_path.prefix, prefixed_path.sub_path,
- });
-
- self.files.addOneAssumeCapacity().* = .{
- .prefixed_path = prefixed_path,
- .contents = null,
- .max_file_size = max_file_size,
- .stat = undefined,
- .bin_digest = undefined,
- };
-
- self.hash.add(prefixed_path.prefix);
- self.hash.addBytes(prefixed_path.sub_path);
-
- return self.files.items.len - 1;
- }
-
- pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void {
- _ = try self.addFile(c_source.src_path, null);
- // Hash the extra flags, with special care to call addFile for file parameters.
- // TODO this logic can likely be improved by utilizing clang_options_data.zig.
- const file_args = [_][]const u8{"-include"};
- var arg_i: usize = 0;
- while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
- const arg = c_source.extra_flags[arg_i];
- self.hash.addBytes(arg);
- for (file_args) |file_arg| {
- if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
- arg_i += 1;
- _ = try self.addFile(c_source.extra_flags[arg_i], null);
- }
- }
- }
- }
-
- pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
- self.hash.add(optional_file_path != null);
- const file_path = optional_file_path orelse return;
- _ = try self.addFile(file_path, null);
- }
-
- pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
- self.hash.add(list_of_files.len);
- for (list_of_files) |file_path| {
- _ = try self.addFile(file_path, null);
- }
- }
-
- /// Check the cache to see if the input exists in it. If it exists, returns `true`.
- /// A hex encoding of its hash is available by calling `final`.
- ///
- /// This function will also acquire an exclusive lock to the manifest file. This means
- /// that a process holding a Manifest will block any other process attempting to
- /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
- /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
- /// file to be locked in exclusive mode.
- ///
- /// The lock on the manifest file is released when `deinit` is called. As another
- /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
- /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
- pub fn hit(self: *Manifest) !bool {
- const gpa = self.cache.gpa;
- assert(self.manifest_file == null);
-
- self.failed_file_index = null;
-
- const ext = ".txt";
- var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
-
- var bin_digest: BinDigest = undefined;
- self.hash.hasher.final(&bin_digest);
-
- _ = std.fmt.bufPrint(
- &self.hex_digest,
- "{s}",
- .{std.fmt.fmtSliceHexLower(&bin_digest)},
- ) catch unreachable;
-
- self.hash.hasher = hasher_init;
- self.hash.hasher.update(&bin_digest);
-
- mem.copy(u8, &manifest_file_path, &self.hex_digest);
- manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
-
- if (self.files.items.len == 0) {
- // If there are no file inputs, we check if the manifest file exists instead of
- // comparing the hashes on the files used for the cached item
- while (true) {
- if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
- .mode = .read_write,
- .lock = .Exclusive,
- .lock_nonblocking = self.want_shared_lock,
- })) |manifest_file| {
- self.manifest_file = manifest_file;
- self.have_exclusive_lock = true;
- break;
- } else |open_err| switch (open_err) {
- error.WouldBlock => {
- self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
- .lock = .Shared,
- });
- break;
- },
- error.FileNotFound => {
- if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
- .read = true,
- .truncate = false,
- .lock = .Exclusive,
- .lock_nonblocking = self.want_shared_lock,
- })) |manifest_file| {
- self.manifest_file = manifest_file;
- self.manifest_dirty = true;
- self.have_exclusive_lock = true;
- return false; // cache miss; exclusive lock already held
- } else |err| switch (err) {
- error.WouldBlock => continue,
- else => |e| return e,
- }
- },
- else => |e| return e,
- }
- }
- } else {
- if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
- .read = true,
- .truncate = false,
- .lock = .Exclusive,
- .lock_nonblocking = self.want_shared_lock,
- })) |manifest_file| {
- self.manifest_file = manifest_file;
- self.have_exclusive_lock = true;
- } else |err| switch (err) {
- error.WouldBlock => {
- self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
- .lock = .Shared,
- });
- },
- else => |e| return e,
- }
- }
-
- self.want_refresh_timestamp = true;
-
- const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
- defer gpa.free(file_contents);
-
- const input_file_count = self.files.items.len;
- var any_file_changed = false;
- var line_iter = mem.tokenize(u8, file_contents, "\n");
- var idx: usize = 0;
- while (line_iter.next()) |line| {
- defer idx += 1;
-
- const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
- const new = try self.files.addOne(gpa);
- new.* = .{
- .prefixed_path = null,
- .contents = null,
- .max_file_size = null,
- .stat = undefined,
- .bin_digest = undefined,
- };
- break :blk new;
- };
-
- var iter = mem.tokenize(u8, line, " ");
- const size = iter.next() orelse return error.InvalidFormat;
- const inode = iter.next() orelse return error.InvalidFormat;
- const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
- const digest_str = iter.next() orelse return error.InvalidFormat;
- const prefix_str = iter.next() orelse return error.InvalidFormat;
- const file_path = iter.rest();
-
- cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
- cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
- cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
- _ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
- const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
- if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
-
- if (file_path.len == 0) {
- return error.InvalidFormat;
- }
- if (cache_hash_file.prefixed_path) |pp| {
- if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
- return error.InvalidFormat;
- }
- }
-
- if (cache_hash_file.prefixed_path == null) {
- cache_hash_file.prefixed_path = .{
- .prefix = prefix,
- .sub_path = try gpa.dupe(u8, file_path),
- };
- }
-
- const pp = cache_hash_file.prefixed_path.?;
- const dir = self.cache.prefixes()[pp.prefix].handle;
- const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
- error.FileNotFound => {
- try self.upgradeToExclusiveLock();
- return false;
- },
- else => return error.CacheUnavailable,
- };
- defer this_file.close();
-
- const actual_stat = this_file.stat() catch |err| {
- self.failed_file_index = idx;
- return err;
- };
- const size_match = actual_stat.size == cache_hash_file.stat.size;
- const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
- const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
-
- if (!size_match or !mtime_match or !inode_match) {
- self.manifest_dirty = true;
-
- cache_hash_file.stat = .{
- .size = actual_stat.size,
- .mtime = actual_stat.mtime,
- .inode = actual_stat.inode,
- };
-
- if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
- // The actual file has an unreliable timestamp, force it to be hashed
- cache_hash_file.stat.mtime = 0;
- cache_hash_file.stat.inode = 0;
- }
-
- var actual_digest: BinDigest = undefined;
- hashFile(this_file, &actual_digest) catch |err| {
- self.failed_file_index = idx;
- return err;
- };
-
- if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
- cache_hash_file.bin_digest = actual_digest;
- // keep going until we have the input file digests
- any_file_changed = true;
- }
- }
-
- if (!any_file_changed) {
- self.hash.hasher.update(&cache_hash_file.bin_digest);
- }
- }
-
- if (any_file_changed) {
- // cache miss
- // keep the manifest file open
- self.unhit(bin_digest, input_file_count);
- try self.upgradeToExclusiveLock();
- return false;
- }
-
- if (idx < input_file_count) {
- self.manifest_dirty = true;
- while (idx < input_file_count) : (idx += 1) {
- const ch_file = &self.files.items[idx];
- self.populateFileHash(ch_file) catch |err| {
- self.failed_file_index = idx;
- return err;
- };
- }
- try self.upgradeToExclusiveLock();
- return false;
- }
-
- if (self.want_shared_lock) {
- try self.downgradeToSharedLock();
- }
-
- return true;
- }
-
- pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
- // Reset the hash.
- self.hash.hasher = hasher_init;
- self.hash.hasher.update(&bin_digest);
-
- // Remove files not in the initial hash.
- for (self.files.items[input_file_count..]) |*file| {
- file.deinit(self.cache.gpa);
- }
- self.files.shrinkRetainingCapacity(input_file_count);
-
- for (self.files.items) |file| {
- self.hash.hasher.update(&file.bin_digest);
- }
- }
-
- fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
- // If the file_time is prior to the most recent problematic timestamp
- // then we don't need to access the filesystem.
- if (file_time < man.recent_problematic_timestamp)
- return false;
-
- // Next we will check the globally shared Cache timestamp, which is accessed
- // from multiple threads.
- man.cache.mutex.lock();
- defer man.cache.mutex.unlock();
-
- // Save the global one to our local one to avoid locking next time.
- man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
- if (file_time < man.recent_problematic_timestamp)
- return false;
-
- // This flag prevents multiple filesystem writes for the same hit() call.
- if (man.want_refresh_timestamp) {
- man.want_refresh_timestamp = false;
-
- var file = man.cache.manifest_dir.createFile("timestamp", .{
- .read = true,
- .truncate = true,
- }) catch return true;
- defer file.close();
-
- // Save locally and also save globally (we still hold the global lock).
- man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
- man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
- }
-
- return file_time >= man.recent_problematic_timestamp;
- }
-
- fn populateFileHash(self: *Manifest, ch_file: *File) !void {
- const pp = ch_file.prefixed_path.?;
- const dir = self.cache.prefixes()[pp.prefix].handle;
- const file = try dir.openFile(pp.sub_path, .{});
- defer file.close();
-
- const actual_stat = try file.stat();
- ch_file.stat = .{
- .size = actual_stat.size,
- .mtime = actual_stat.mtime,
- .inode = actual_stat.inode,
- };
-
- if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
- // The actual file has an unreliable timestamp, force it to be hashed
- ch_file.stat.mtime = 0;
- ch_file.stat.inode = 0;
- }
-
- if (ch_file.max_file_size) |max_file_size| {
- if (ch_file.stat.size > max_file_size) {
- return error.FileTooBig;
- }
-
- const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
- errdefer self.cache.gpa.free(contents);
-
- // Hash while reading from disk, to keep the contents in the cpu cache while
- // doing hashing.
- var hasher = hasher_init;
- var off: usize = 0;
- while (true) {
- // give me everything you've got, captain
- const bytes_read = try file.read(contents[off..]);
- if (bytes_read == 0) break;
- hasher.update(contents[off..][0..bytes_read]);
- off += bytes_read;
- }
- hasher.final(&ch_file.bin_digest);
-
- ch_file.contents = contents;
- } else {
- try hashFile(file, &ch_file.bin_digest);
- }
-
- self.hash.hasher.update(&ch_file.bin_digest);
- }
-
- /// Add a file as a dependency of process being cached, after the initial hash has been
- /// calculated. This is useful for processes that don't know all the files that
- /// are depended on ahead of time. For example, a source file that can import other files
- /// will need to be recompiled if the imported file is changed.
- pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
- assert(self.manifest_file != null);
-
- const gpa = self.cache.gpa;
- const prefixed_path = try self.cache.findPrefix(file_path);
- errdefer gpa.free(prefixed_path.sub_path);
-
- log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{
- file_path, prefixed_path.prefix, prefixed_path.sub_path,
- });
-
- const new_ch_file = try self.files.addOne(gpa);
- new_ch_file.* = .{
- .prefixed_path = prefixed_path,
- .max_file_size = max_file_size,
- .stat = undefined,
- .bin_digest = undefined,
- .contents = null,
- };
- errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
-
- try self.populateFileHash(new_ch_file);
-
- return new_ch_file.contents.?;
- }
-
- /// Add a file as a dependency of process being cached, after the initial hash has been
- /// calculated. This is useful for processes that don't know the all the files that
- /// are depended on ahead of time. For example, a source file that can import other files
- /// will need to be recompiled if the imported file is changed.
- pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
- assert(self.manifest_file != null);
-
- const gpa = self.cache.gpa;
- const prefixed_path = try self.cache.findPrefix(file_path);
- errdefer gpa.free(prefixed_path.sub_path);
-
- log.debug("Manifest.addFilePost {s} -> {d} {s}", .{
- file_path, prefixed_path.prefix, prefixed_path.sub_path,
- });
-
- const new_ch_file = try self.files.addOne(gpa);
- new_ch_file.* = .{
- .prefixed_path = prefixed_path,
- .max_file_size = null,
- .stat = undefined,
- .bin_digest = undefined,
- .contents = null,
- };
- errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
-
- try self.populateFileHash(new_ch_file);
- }
-
- /// Like `addFilePost` but when the file contents have already been loaded from disk.
- /// On success, cache takes ownership of `resolved_path`.
- pub fn addFilePostContents(
- self: *Manifest,
- resolved_path: []u8,
- bytes: []const u8,
- stat: File.Stat,
- ) error{OutOfMemory}!void {
- assert(self.manifest_file != null);
- const gpa = self.cache.gpa;
-
- const ch_file = try self.files.addOne(gpa);
- errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
-
- log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path});
-
- const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
- errdefer gpa.free(prefixed_path.sub_path);
-
- log.debug("Manifest.addFilePostContents -> {d} {s}", .{
- prefixed_path.prefix, prefixed_path.sub_path,
- });
-
- ch_file.* = .{
- .prefixed_path = prefixed_path,
- .max_file_size = null,
- .stat = stat,
- .bin_digest = undefined,
- .contents = null,
- };
-
- if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
- // The actual file has an unreliable timestamp, force it to be hashed
- ch_file.stat.mtime = 0;
- ch_file.stat.inode = 0;
- }
-
- {
- var hasher = hasher_init;
- hasher.update(bytes);
- hasher.final(&ch_file.bin_digest);
- }
-
- self.hash.hasher.update(&ch_file.bin_digest);
- }
-
- pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
- assert(self.manifest_file != null);
-
- const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
- defer self.cache.gpa.free(dep_file_contents);
-
- var error_buf = std.ArrayList(u8).init(self.cache.gpa);
- defer error_buf.deinit();
-
- var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents };
-
- // Skip first token: target.
- switch (it.next() orelse return) { // Empty dep file OK.
- .target, .target_must_resolve, .prereq => {},
- else => |err| {
- try err.printError(error_buf.writer());
- log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
- return error.InvalidDepFile;
- },
- }
- // Process 0+ preqreqs.
- // Clang is invoked in single-source mode so we never get more targets.
- while (true) {
- switch (it.next() orelse return) {
- .target, .target_must_resolve => return,
- .prereq => |file_path| try self.addFilePost(file_path),
- else => |err| {
- try err.printError(error_buf.writer());
- log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
- return error.InvalidDepFile;
- },
- }
- }
- }
-
- /// Returns a hex encoded hash of the inputs.
- pub fn final(self: *Manifest) [hex_digest_len]u8 {
- assert(self.manifest_file != null);
-
- // We don't close the manifest file yet, because we want to
- // keep it locked until the API user is done using it.
- // We also don't write out the manifest yet, because until
- // cache_release is called we still might be working on creating
- // the artifacts to cache.
-
- var bin_digest: BinDigest = undefined;
- self.hash.hasher.final(&bin_digest);
-
- var out_digest: [hex_digest_len]u8 = undefined;
- _ = std.fmt.bufPrint(
- &out_digest,
- "{s}",
- .{std.fmt.fmtSliceHexLower(&bin_digest)},
- ) catch unreachable;
-
- return out_digest;
- }
-
- /// If `want_shared_lock` is true, this function automatically downgrades the
- /// lock from exclusive to shared.
- pub fn writeManifest(self: *Manifest) !void {
- assert(self.have_exclusive_lock);
-
- const manifest_file = self.manifest_file.?;
- if (self.manifest_dirty) {
- self.manifest_dirty = false;
-
- var contents = std.ArrayList(u8).init(self.cache.gpa);
- defer contents.deinit();
-
- const writer = contents.writer();
- var encoded_digest: [hex_digest_len]u8 = undefined;
-
- for (self.files.items) |file| {
- _ = std.fmt.bufPrint(
- &encoded_digest,
- "{s}",
- .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
- ) catch unreachable;
- try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
- file.stat.size,
- file.stat.inode,
- file.stat.mtime,
- &encoded_digest,
- file.prefixed_path.?.prefix,
- file.prefixed_path.?.sub_path,
- });
- }
-
- try manifest_file.setEndPos(contents.items.len);
- try manifest_file.pwriteAll(contents.items, 0);
- }
-
- if (self.want_shared_lock) {
- try self.downgradeToSharedLock();
- }
- }
-
- fn downgradeToSharedLock(self: *Manifest) !void {
- if (!self.have_exclusive_lock) return;
-
- // WASI does not currently support flock, so we bypass it here.
- // TODO: If/when flock is supported on WASI, this check should be removed.
- // See https://github.com/WebAssembly/wasi-filesystem/issues/2
- if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
- const manifest_file = self.manifest_file.?;
- try manifest_file.downgradeLock();
- }
-
- self.have_exclusive_lock = false;
- }
-
- fn upgradeToExclusiveLock(self: *Manifest) !void {
- if (self.have_exclusive_lock) return;
- assert(self.manifest_file != null);
-
- // WASI does not currently support flock, so we bypass it here.
- // TODO: If/when flock is supported on WASI, this check should be removed.
- // See https://github.com/WebAssembly/wasi-filesystem/issues/2
- if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
- const manifest_file = self.manifest_file.?;
- // Here we intentionally have a period where the lock is released, in case there are
- // other processes holding a shared lock.
- manifest_file.unlock();
- try manifest_file.lock(.Exclusive);
- }
- self.have_exclusive_lock = true;
- }
-
- /// Obtain only the data needed to maintain a lock on the manifest file.
- /// The `Manifest` remains safe to deinit.
- /// Don't forget to call `writeManifest` before this!
- pub fn toOwnedLock(self: *Manifest) Lock {
- const lock: Lock = .{
- .manifest_file = self.manifest_file.?,
- };
-
- self.manifest_file = null;
- return lock;
- }
-
- /// Releases the manifest file and frees any memory the Manifest was using.
- /// `Manifest.hit` must be called first.
- /// Don't forget to call `writeManifest` before this!
- pub fn deinit(self: *Manifest) void {
- if (self.manifest_file) |file| {
- if (builtin.os.tag == .windows) {
- // See Lock.release for why this is required on Windows
- file.unlock();
- }
-
- file.close();
- }
- for (self.files.items) |*file| {
- file.deinit(self.cache.gpa);
- }
- self.files.deinit(self.cache.gpa);
- }
-};
-
-/// On operating systems that support symlinks, does a readlink. On other operating systems,
-/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
-/// it is treated as not supporting symlinks.
-pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
- if (builtin.os.tag == .windows) {
- return dir.readFile(sub_path, buffer);
- } else {
- return dir.readLink(sub_path, buffer);
- }
-}
-
-/// On operating systems that support symlinks, does a symlink. On other operating systems,
-/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
-/// it is treated as not supporting symlinks.
-/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
-pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
- assert(data.len <= 255);
- if (builtin.os.tag == .windows) {
- return dir.writeFile(sub_path, data);
- } else {
- return dir.symLink(data, sub_path, .{});
- }
-}
-
-fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
- var buf: [1024]u8 = undefined;
-
- var hasher = hasher_init;
- while (true) {
- const bytes_read = try file.read(&buf);
- if (bytes_read == 0) break;
- hasher.update(buf[0..bytes_read]);
- }
-
- hasher.final(bin_digest);
-}
-
-// Create/Write a file, close it, then grab its stat.mtime timestamp.
-fn testGetCurrentFileTimestamp() !i128 {
- var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
- .read = true,
- .truncate = true,
- });
- defer file.close();
-
- return (try file.stat()).mtime;
-}
-
-test "cache file and then recall it" {
- if (builtin.os.tag == .wasi) {
- // https://github.com/ziglang/zig/issues/5437
- return error.SkipZigTest;
- }
-
- const cwd = fs.cwd();
-
- const temp_file = "test.txt";
- const temp_manifest_dir = "temp_manifest_dir";
-
- try cwd.writeFile(temp_file, "Hello, world!\n");
-
- // Wait for file timestamps to tick
- const initial_time = try testGetCurrentFileTimestamp();
- while ((try testGetCurrentFileTimestamp()) == initial_time) {
- std.time.sleep(1);
- }
-
- var digest1: [hex_digest_len]u8 = undefined;
- var digest2: [hex_digest_len]u8 = undefined;
-
- {
- var cache = Cache{
- .gpa = testing.allocator,
- .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
- };
- cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
- defer cache.manifest_dir.close();
-
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.add(true);
- ch.hash.add(@as(u16, 1234));
- ch.hash.addBytes("1234");
- _ = try ch.addFile(temp_file, null);
-
- // There should be nothing in the cache
- try testing.expectEqual(false, try ch.hit());
-
- digest1 = ch.final();
- try ch.writeManifest();
- }
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.add(true);
- ch.hash.add(@as(u16, 1234));
- ch.hash.addBytes("1234");
- _ = try ch.addFile(temp_file, null);
-
- // Cache hit! We just "built" the same file
- try testing.expect(try ch.hit());
- digest2 = ch.final();
-
- try testing.expectEqual(false, ch.have_exclusive_lock);
- }
-
- try testing.expectEqual(digest1, digest2);
- }
-
- try cwd.deleteTree(temp_manifest_dir);
- try cwd.deleteFile(temp_file);
-}
-
-test "check that changing a file makes cache fail" {
- if (builtin.os.tag == .wasi) {
- // https://github.com/ziglang/zig/issues/5437
- return error.SkipZigTest;
- }
- const cwd = fs.cwd();
-
- const temp_file = "cache_hash_change_file_test.txt";
- const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
- const original_temp_file_contents = "Hello, world!\n";
- const updated_temp_file_contents = "Hello, world; but updated!\n";
-
- try cwd.deleteTree(temp_manifest_dir);
- try cwd.deleteTree(temp_file);
-
- try cwd.writeFile(temp_file, original_temp_file_contents);
-
- // Wait for file timestamps to tick
- const initial_time = try testGetCurrentFileTimestamp();
- while ((try testGetCurrentFileTimestamp()) == initial_time) {
- std.time.sleep(1);
- }
-
- var digest1: [hex_digest_len]u8 = undefined;
- var digest2: [hex_digest_len]u8 = undefined;
-
- {
- var cache = Cache{
- .gpa = testing.allocator,
- .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
- };
- cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
- defer cache.manifest_dir.close();
-
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.addBytes("1234");
- const temp_file_idx = try ch.addFile(temp_file, 100);
-
- // There should be nothing in the cache
- try testing.expectEqual(false, try ch.hit());
-
- try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
-
- digest1 = ch.final();
-
- try ch.writeManifest();
- }
-
- try cwd.writeFile(temp_file, updated_temp_file_contents);
-
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.addBytes("1234");
- const temp_file_idx = try ch.addFile(temp_file, 100);
-
- // A file that we depend on has been updated, so the cache should not contain an entry for it
- try testing.expectEqual(false, try ch.hit());
-
- // The cache system does not keep the contents of re-hashed input files.
- try testing.expect(ch.files.items[temp_file_idx].contents == null);
-
- digest2 = ch.final();
-
- try ch.writeManifest();
- }
-
- try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
- }
-
- try cwd.deleteTree(temp_manifest_dir);
- try cwd.deleteTree(temp_file);
-}
-
-test "no file inputs" {
- if (builtin.os.tag == .wasi) {
- // https://github.com/ziglang/zig/issues/5437
- return error.SkipZigTest;
- }
- const cwd = fs.cwd();
- const temp_manifest_dir = "no_file_inputs_manifest_dir";
- defer cwd.deleteTree(temp_manifest_dir) catch {};
-
- var digest1: [hex_digest_len]u8 = undefined;
- var digest2: [hex_digest_len]u8 = undefined;
-
- var cache = Cache{
- .gpa = testing.allocator,
- .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
- };
- cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
- defer cache.manifest_dir.close();
-
- {
- var man = cache.obtain();
- defer man.deinit();
-
- man.hash.addBytes("1234");
-
- // There should be nothing in the cache
- try testing.expectEqual(false, try man.hit());
-
- digest1 = man.final();
-
- try man.writeManifest();
- }
- {
- var man = cache.obtain();
- defer man.deinit();
-
- man.hash.addBytes("1234");
-
- try testing.expect(try man.hit());
- digest2 = man.final();
- try testing.expectEqual(false, man.have_exclusive_lock);
- }
-
- try testing.expectEqual(digest1, digest2);
-}
-
-test "Manifest with files added after initial hash work" {
- if (builtin.os.tag == .wasi) {
- // https://github.com/ziglang/zig/issues/5437
- return error.SkipZigTest;
- }
- const cwd = fs.cwd();
-
- const temp_file1 = "cache_hash_post_file_test1.txt";
- const temp_file2 = "cache_hash_post_file_test2.txt";
- const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
-
- try cwd.writeFile(temp_file1, "Hello, world!\n");
- try cwd.writeFile(temp_file2, "Hello world the second!\n");
-
- // Wait for file timestamps to tick
- const initial_time = try testGetCurrentFileTimestamp();
- while ((try testGetCurrentFileTimestamp()) == initial_time) {
- std.time.sleep(1);
- }
-
- var digest1: [hex_digest_len]u8 = undefined;
- var digest2: [hex_digest_len]u8 = undefined;
- var digest3: [hex_digest_len]u8 = undefined;
-
- {
- var cache = Cache{
- .gpa = testing.allocator,
- .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
- };
- cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
- defer cache.manifest_dir.close();
-
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.addBytes("1234");
- _ = try ch.addFile(temp_file1, null);
-
- // There should be nothing in the cache
- try testing.expectEqual(false, try ch.hit());
-
- _ = try ch.addFilePost(temp_file2);
-
- digest1 = ch.final();
- try ch.writeManifest();
- }
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.addBytes("1234");
- _ = try ch.addFile(temp_file1, null);
-
- try testing.expect(try ch.hit());
- digest2 = ch.final();
-
- try testing.expectEqual(false, ch.have_exclusive_lock);
- }
- try testing.expect(mem.eql(u8, &digest1, &digest2));
-
- // Modify the file added after initial hash
- try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
-
- // Wait for file timestamps to tick
- const initial_time2 = try testGetCurrentFileTimestamp();
- while ((try testGetCurrentFileTimestamp()) == initial_time2) {
- std.time.sleep(1);
- }
-
- {
- var ch = cache.obtain();
- defer ch.deinit();
-
- ch.hash.addBytes("1234");
- _ = try ch.addFile(temp_file1, null);
-
- // A file that we depend on has been updated, so the cache should not contain an entry for it
- try testing.expectEqual(false, try ch.hit());
-
- _ = try ch.addFilePost(temp_file2);
-
- digest3 = ch.final();
-
- try ch.writeManifest();
- }
-
- try testing.expect(!mem.eql(u8, &digest1, &digest3));
- }
-
- try cwd.deleteTree(temp_manifest_dir);
- try cwd.deleteFile(temp_file1);
- try cwd.deleteFile(temp_file2);
-}
diff --git a/src/Compilation.zig b/src/Compilation.zig
@@ -26,7 +26,7 @@ const wasi_libc = @import("wasi_libc.zig");
const fatal = @import("main.zig").fatal;
const clangMain = @import("main.zig").clangMain;
const Module = @import("Module.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
const translate_c = @import("translate_c.zig");
const clang = @import("clang.zig");
const c_codegen = @import("codegen/c.zig");
@@ -807,44 +807,7 @@ pub const AllErrors = struct {
}
};
-pub const Directory = struct {
- /// This field is redundant for operations that can act on the open directory handle
- /// directly, but it is needed when passing the directory to a child process.
- /// `null` means cwd.
- path: ?[]const u8,
- handle: std.fs.Dir,
-
- pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
- if (self.path) |p| {
- // TODO clean way to do this with only 1 allocation
- const part2 = try std.fs.path.join(allocator, paths);
- defer allocator.free(part2);
- return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
- } else {
- return std.fs.path.join(allocator, paths);
- }
- }
-
- pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
- if (self.path) |p| {
- // TODO clean way to do this with only 1 allocation
- const part2 = try std.fs.path.join(allocator, paths);
- defer allocator.free(part2);
- return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
- } else {
- return std.fs.path.joinZ(allocator, paths);
- }
- }
-
- /// Whether or not the handle should be closed, or the path should be freed
- /// is determined by usage, however this function is provided for convenience
- /// if it happens to be what the caller needs.
- pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
- self.handle.close();
- if (self.path) |p| gpa.free(p);
- self.* = undefined;
- }
-};
+pub const Directory = Cache.Directory;
pub const EmitLoc = struct {
/// If this is `null` it means the file will be output to the cache directory.
@@ -854,6 +817,35 @@ pub const EmitLoc = struct {
basename: []const u8,
};
+pub const cache_helpers = struct {
+ pub fn addEmitLoc(hh: *Cache.HashHelper, emit_loc: EmitLoc) void {
+ hh.addBytes(emit_loc.basename);
+ }
+
+ pub fn addOptionalEmitLoc(hh: *Cache.HashHelper, optional_emit_loc: ?EmitLoc) void {
+ hh.add(optional_emit_loc != null);
+ addEmitLoc(hh, optional_emit_loc orelse return);
+ }
+
+ pub fn hashCSource(self: *Cache.Manifest, c_source: Compilation.CSourceFile) !void {
+ _ = try self.addFile(c_source.src_path, null);
+ // Hash the extra flags, with special care to call addFile for file parameters.
+ // TODO this logic can likely be improved by utilizing clang_options_data.zig.
+ const file_args = [_][]const u8{"-include"};
+ var arg_i: usize = 0;
+ while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
+ const arg = c_source.extra_flags[arg_i];
+ self.hash.addBytes(arg);
+ for (file_args) |file_arg| {
+ if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
+ arg_i += 1;
+ _ = try self.addFile(c_source.extra_flags[arg_i], null);
+ }
+ }
+ }
+ }
+};
+
pub const ClangPreprocessorMode = enum {
no,
/// This means we are doing `zig cc -E -o <path>`.
@@ -1523,8 +1515,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
cache.hash.add(link_libunwind);
cache.hash.add(options.output_mode);
cache.hash.add(options.machine_code_model);
- cache.hash.addOptionalEmitLoc(options.emit_bin);
- cache.hash.addOptionalEmitLoc(options.emit_implib);
+ cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_bin);
+ cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_implib);
cache.hash.addBytes(options.root_name);
if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
// TODO audit this and make sure everything is in it
@@ -2638,11 +2630,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
man.hash.addListOfBytes(key.src.extra_flags);
}
- man.hash.addOptionalEmitLoc(comp.emit_asm);
- man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
- man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
- man.hash.addOptionalEmitLoc(comp.emit_analysis);
- man.hash.addOptionalEmitLoc(comp.emit_docs);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_analysis);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_docs);
man.hash.addListOfBytes(comp.clang_argv);
@@ -3961,11 +3953,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
defer man.deinit();
man.hash.add(comp.clang_preprocessor_mode);
- man.hash.addOptionalEmitLoc(comp.emit_asm);
- man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
- man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
+ cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
- try man.hashCSource(c_object.src);
+ try cache_helpers.hashCSource(&man, c_object.src);
var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
defer arena_allocator.deinit();
diff --git a/src/Module.zig b/src/Module.zig
@@ -16,7 +16,7 @@ const Ast = std.zig.Ast;
const Module = @This();
const Compilation = @import("Compilation.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
const Value = @import("value.zig").Value;
const Type = @import("type.zig").Type;
const TypedValue = @import("TypedValue.zig");
diff --git a/src/Package.zig b/src/Package.zig
@@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig");
const Module = @import("Module.zig");
const ThreadPool = @import("ThreadPool.zig");
const WaitGroup = @import("WaitGroup.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
const build_options = @import("build_options");
const Manifest = @import("Manifest.zig");
diff --git a/src/glibc.zig b/src/glibc.zig
@@ -11,7 +11,7 @@ const target_util = @import("target.zig");
const Compilation = @import("Compilation.zig");
const build_options = @import("build_options");
const trace = @import("tracy.zig").trace;
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
const Package = @import("Package.zig");
pub const Lib = struct {
diff --git a/src/link.zig b/src/link.zig
@@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig");
const Air = @import("Air.zig");
const Allocator = std.mem.Allocator;
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
const Compilation = @import("Compilation.zig");
const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
const Liveness = @import("Liveness.zig");
diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig
@@ -5,6 +5,7 @@ const assert = std.debug.assert;
const fs = std.fs;
const log = std.log.scoped(.link);
const mem = std.mem;
+const Cache = std.Build.Cache;
const mingw = @import("../../mingw.zig");
const link = @import("../../link.zig");
@@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace;
const Allocator = mem.Allocator;
-const Cache = @import("../../Cache.zig");
const Coff = @import("../Coff.zig");
const Compilation = @import("../../Compilation.zig");
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
@@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace;
const Air = @import("../Air.zig");
const Allocator = std.mem.Allocator;
pub const Atom = @import("Elf/Atom.zig");
-const Cache = @import("../Cache.zig");
+const Cache = std.Build.Cache;
const Compilation = @import("../Compilation.zig");
const Dwarf = @import("Dwarf.zig");
const File = link.File;
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
@@ -28,7 +28,7 @@ const Air = @import("../Air.zig");
const Allocator = mem.Allocator;
const Archive = @import("MachO/Archive.zig");
pub const Atom = @import("MachO/Atom.zig");
-const Cache = @import("../Cache.zig");
+const Cache = std.Build.Cache;
const CodeSignature = @import("MachO/CodeSignature.zig");
const Compilation = @import("../Compilation.zig");
const Dwarf = File.Dwarf;
diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig
@@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace;
const Allocator = mem.Allocator;
const Archive = @import("Archive.zig");
const Atom = @import("ZldAtom.zig");
-const Cache = @import("../../Cache.zig");
+const Cache = std.Build.Cache;
const CodeSignature = @import("CodeSignature.zig");
const Compilation = @import("../../Compilation.zig");
const DwarfInfo = @import("DwarfInfo.zig");
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
@@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain;
const trace = @import("../tracy.zig").trace;
const build_options = @import("build_options");
const wasi_libc = @import("../wasi_libc.zig");
-const Cache = @import("../Cache.zig");
+const Cache = std.Build.Cache;
const Type = @import("../type.zig").Type;
const TypedValue = @import("../TypedValue.zig");
const LlvmObject = @import("../codegen/llvm.zig").Object;
diff --git a/src/main.zig b/src/main.zig
@@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
const wasi_libc = @import("wasi_libc.zig");
const translate_c = @import("translate_c.zig");
const clang = @import("clang.zig");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
const target_util = @import("target.zig");
const ThreadPool = @import("ThreadPool.zig");
const crash_report = @import("crash_report.zig");
@@ -3615,7 +3615,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void
defer if (enable_cache) man.deinit();
man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects
- man.hashCSource(c_source_file) catch |err| {
+ Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| {
fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) });
};
diff --git a/src/mingw.zig b/src/mingw.zig
@@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw);
const builtin = @import("builtin");
const Compilation = @import("Compilation.zig");
const build_options = @import("build_options");
-const Cache = @import("Cache.zig");
+const Cache = std.Build.Cache;
pub const CRTFile = enum {
crt2_o,
diff --git a/test/tests.zig b/test/tests.zig
@@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co
const run_cmd = exe.run();
run_cmd.addArgs(&[_][]const u8{
fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,
- b.pathFromRoot(b.cache_root),
+ b.pathFromRoot(b.cache_root.path orelse "."),
});
step.dependOn(&run_cmd.step);
@@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {
}
var zig_args = ArrayList([]const u8).init(b.allocator);
- const rel_zig_exe = fs.path.relative(b.allocator, b.build_root, b.zig_exe) catch unreachable;
+ const rel_zig_exe = fs.path.relative(b.allocator, b.build_root.path orelse ".", b.zig_exe) catch unreachable;
zig_args.append(rel_zig_exe) catch unreachable;
zig_args.append("build") catch unreachable;
