motiejus/zig
1
Fork 0
Code Releases Activity

wasm: consolidate flushModule and linkWithZld

Browse Source 
We now use a single function to use the in-house WebAssembly linker
rather than wasm-ld. For both incremental compilation and traditional
linking we use the same codepath.
This commit is contained in:
Luuk de Gram
2024-02-07 06:57:32 +01:00
parent 5aec88fa41
commit c99ef23862
2 changed files with 33 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

327
src/link/Wasm.zig
View File

@@ -177,10 +177,6 @@ undefs: std.AutoArrayHashMapUnmanaged(u32, SymbolLoc) = .{},
/// data of a symbol, such as its size, or its offset to perform a relocation.
/// Undefined (and synthetic) symbols do not have an Atom and therefore cannot be mapped.
symbol_atom: std.AutoHashMapUnmanaged(SymbolLoc, Atom.Index) = .{},
/// Maps a symbol's location to its export name, which may differ from the decl's name
/// which does the exporting.
/// Note: The value represents the offset into the string table, rather than the actual string.
export_names: std.AutoHashMapUnmanaged(SymbolLoc, u32) = .{},
/// List of atom indexes of functions that are generated by the backend,
/// rather than by the linker.
@@ -1398,7 +1394,6 @@ pub fn deinit(wasm: *Wasm) void {
wasm.undefs.deinit(gpa);
wasm.discarded.deinit(gpa);
wasm.symbol_atom.deinit(gpa);
wasm.export_names.deinit(gpa);
wasm.atoms.deinit(gpa);
wasm.managed_atoms.deinit(gpa);
wasm.segments.deinit(gpa);
@@ -2133,10 +2128,10 @@ fn setupExports(wasm: *Wasm) !void {
if (!symbol.isExported(comp.config.rdynamic)) continue;
const sym_name = sym_loc.getName(wasm);
const export_name = if (wasm.export_names.get(sym_loc)) |name| name else blk: {
if (sym_loc.file == .null) break :blk symbol.name;
break :blk try wasm.string_table.put(gpa, sym_name);
};
const export_name = if (sym_loc.file == .null)
symbol.name
else
try wasm.string_table.put(gpa, sym_name);
const exp: types.Export = if (symbol.tag == .data) exp: {
const global_index = @as(u32, @intCast(wasm.imported_globals_count + wasm.wasm_globals.items.len));
try wasm.wasm_globals.append(gpa, .{
@@ -2437,168 +2432,45 @@ fn appendDummySegment(wasm: *Wasm) !void {
});
}
fn resetState(wasm: *Wasm) void {
const gpa = wasm.base.comp.gpa;
for (wasm.segment_info.values()) |segment_info| {
gpa.free(segment_info.name);
}
// TODO: Revisit
// var atom_it = wasm.decls.valueIterator();
// while (atom_it.next()) |atom_index| {
// const atom = wasm.getAtomPtr(atom_index.*);
// atom.next = null;
// atom.prev = null;
// for (atom.locals.items) |local_atom_index| {
// const local_atom = wasm.getAtomPtr(local_atom_index);
// local_atom.next = null;
// local_atom.prev = null;
// }
// }
wasm.functions.clearRetainingCapacity();
wasm.exports.clearRetainingCapacity();
wasm.segments.clearRetainingCapacity();
wasm.segment_info.clearRetainingCapacity();
wasm.data_segments.clearRetainingCapacity();
wasm.atoms.clearRetainingCapacity();
wasm.symbol_atom.clearRetainingCapacity();
wasm.code_section_index = null;
wasm.debug_info_index = null;
wasm.debug_line_index = null;
wasm.debug_loc_index = null;
wasm.debug_str_index = null;
wasm.debug_ranges_index = null;
wasm.debug_abbrev_index = null;
wasm.debug_pubnames_index = null;
wasm.debug_pubtypes_index = null;
}
pub fn flush(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) link.File.FlushError!void {
const comp = wasm.base.comp;
const use_lld = build_options.have_llvm and comp.config.use_lld;
const use_llvm = comp.config.use_llvm;
if (use_lld) {
return wasm.linkWithLLD(arena, prog_node);
} else if (use_llvm) {
return wasm.linkWithZld(arena, prog_node);
} else {
return wasm.flushModule(arena, prog_node);
}
return wasm.flushModule(arena, prog_node);
}
/// Uses the in-house linker to link one or multiple object -and archive files into a WebAssembly binary.
fn linkWithZld(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) link.File.FlushError!void {
pub fn flushModule(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) link.File.FlushError!void {
const tracy = trace(@src());
defer tracy.end();
const comp = wasm.base.comp;
const shared_memory = comp.config.shared_memory;
const import_memory = comp.config.import_memory;
const directory = wasm.base.emit.directory; // Just an alias to make it shorter to type.
const full_out_path = try directory.join(arena, &[_][]const u8{wasm.base.emit.sub_path});
const opt_zcu = comp.module;
const use_llvm = comp.config.use_llvm;
// If there is no Zig code to compile, then we should skip flushing the output file because it
// will not be part of the linker line anyway.
const module_obj_path: ?[]const u8 = if (opt_zcu != null) blk: {
assert(use_llvm); // `linkWithZld` should never be called when the Wasm backend is used
try wasm.flushModule(arena, prog_node);
if (fs.path.dirname(full_out_path)) |dirname| {
break :blk try fs.path.join(arena, &.{ dirname, wasm.base.zcu_object_sub_path.? });
} else {
break :blk wasm.base.zcu_object_sub_path.?;
}
} else null;
if (wasm.llvm_object) |llvm_object| {
try wasm.base.emitLlvmObject(arena, llvm_object, prog_node);
const use_lld = build_options.have_llvm and comp.config.use_lld;
if (use_lld) return;
}
var sub_prog_node = prog_node.start("Wasm Flush", 0);
sub_prog_node.activate();
defer sub_prog_node.end();
const compiler_rt_path: ?[]const u8 = blk: {
if (comp.compiler_rt_obj) |obj| break :blk obj.full_object_path;
if (comp.compiler_rt_lib) |lib| break :blk lib.full_object_path;
break :blk null;
};
const id_symlink_basename = "zld.id";
var man: Cache.Manifest = undefined;
defer if (!wasm.base.disable_lld_caching) man.deinit();
var digest: [Cache.hex_digest_len]u8 = undefined;
const objects = comp.objects;
// NOTE: The following section must be maintained to be equal
// as the section defined in `linkWithLLD`
if (!wasm.base.disable_lld_caching) {
man = comp.cache_parent.obtain();
// We are about to obtain this lock, so here we give other processes a chance first.
wasm.base.releaseLock();
comptime assert(Compilation.link_hash_implementation_version == 12);
for (objects) |obj| {
_ = try man.addFile(obj.path, null);
man.hash.add(obj.must_link);
const directory = wasm.base.emit.directory; // Just an alias to make it shorter to type.
const full_out_path = try directory.join(arena, &[_][]const u8{wasm.base.emit.sub_path});
const module_obj_path: ?[]const u8 = if (wasm.base.zcu_object_sub_path) |path| blk: {
if (fs.path.dirname(full_out_path)) |dirname| {
break :blk try fs.path.join(arena, &.{ dirname, path });
} else {
break :blk path;
}
for (comp.c_object_table.keys()) |key| {
_ = try man.addFile(key.status.success.object_path, null);
}
try man.addOptionalFile(module_obj_path);
try man.addOptionalFile(compiler_rt_path);
man.hash.addOptionalBytes(wasm.entry_name);
man.hash.add(wasm.base.stack_size);
man.hash.add(wasm.base.build_id);
man.hash.add(import_memory);
man.hash.add(shared_memory);
man.hash.add(wasm.import_table);
man.hash.add(wasm.export_table);
man.hash.addOptional(wasm.initial_memory);
man.hash.addOptional(wasm.max_memory);
man.hash.addOptional(wasm.global_base);
man.hash.addListOfBytes(wasm.export_symbol_names);
// strip does not need to go into the linker hash because it is part of the hash namespace
// We don't actually care whether it's a cache hit or miss; we just need the digest and the lock.
_ = try man.hit();
digest = man.final();
var prev_digest_buf: [digest.len]u8 = undefined;
const prev_digest: []u8 = Cache.readSmallFile(
directory.handle,
id_symlink_basename,
&prev_digest_buf,
) catch |err| blk: {
log.debug("WASM LLD new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });
// Handle this as a cache miss.
break :blk prev_digest_buf[0..0];
};
if (mem.eql(u8, prev_digest, &digest)) {
log.debug("WASM LLD digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
// Hot diggity dog! The output binary is already there.
wasm.base.lock = man.toOwnedLock();
return;
}
log.debug("WASM LLD prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) });
// We are about to change the output file to be different, so we invalidate the build hash now.
directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
error.FileNotFound => {},
else => |e| return e,
};
}
} else null;
// Positional arguments to the linker such as object files and static archives.
var positionals = std.ArrayList([]const u8).init(arena);
try positionals.ensureUnusedCapacity(objects.len);
try positionals.ensureUnusedCapacity(comp.objects.len);
const target = comp.root_mod.resolved_target.result;
const output_mode = comp.config.output_mode;
@@ -2607,6 +2479,10 @@ fn linkWithZld(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) lin
const link_libcpp = comp.config.link_libcpp;
const wasi_exec_model = comp.config.wasi_exec_model;
if (wasm.zigObjectPtr()) |zig_object| {
try zig_object.flushModule(wasm);
}
// When the target os is WASI, we allow linking with WASI-LIBC
if (target.os.tag == .wasi) {
const is_exe_or_dyn_lib = output_mode == .Exe or
@@ -2638,7 +2514,7 @@ fn linkWithZld(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) lin
try positionals.append(path);
}
for (objects) |object| {
for (comp.objects) |object| {
try positionals.append(object.path);
}
@@ -2651,93 +2527,6 @@ fn linkWithZld(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) lin
try wasm.parseInputFiles(positionals.items);
for (wasm.objects.items) |object_index| {
try wasm.resolveSymbolsInObject(object_index);
}
var emit_features_count: u32 = 0;
var enabled_features: [@typeInfo(types.Feature.Tag).Enum.fields.len]bool = undefined;
try wasm.validateFeatures(&enabled_features, &emit_features_count);
try wasm.resolveSymbolsInArchives();
try wasm.resolveLazySymbols();
try wasm.checkUndefinedSymbols();
try wasm.setupInitFunctions();
try wasm.setupStart();
try wasm.markReferences();
try wasm.setupImports();
try wasm.mergeSections();
try wasm.mergeTypes();
try wasm.allocateAtoms();
try wasm.setupMemory();
wasm.allocateVirtualAddresses();
wasm.mapFunctionTable();
try wasm.initializeCallCtorsFunction();
try wasm.setupInitMemoryFunction();
try wasm.setupTLSRelocationsFunction();
try wasm.initializeTLSFunction();
try wasm.setupStartSection();
try wasm.setupExports();
try wasm.writeToFile(enabled_features, emit_features_count, arena);
if (!wasm.base.disable_lld_caching) {
// Update the file with the digest. If it fails we can continue; it only
// means that the next invocation will have an unnecessary cache miss.
Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
log.warn("failed to save linking hash digest symlink: {s}", .{@errorName(err)});
};
// Again failure here only means an unnecessary cache miss.
man.writeManifest() catch |err| {
log.warn("failed to write cache manifest when linking: {s}", .{@errorName(err)});
};
// We hang on to this lock so that the output file path can be used without
// other processes clobbering it.
wasm.base.lock = man.toOwnedLock();
}
}
pub fn flushModule(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node) link.File.FlushError!void {
const tracy = trace(@src());
defer tracy.end();
const comp = wasm.base.comp;
if (wasm.llvm_object) |llvm_object| {
try wasm.base.emitLlvmObject(arena, llvm_object, prog_node);
return;
}
var sub_prog_node = prog_node.start("Wasm Flush", 0);
sub_prog_node.activate();
defer sub_prog_node.end();
if (wasm.zigObjectPtr()) |zig_object| {
try zig_object.flushModule(wasm);
}
// ensure the error names table is populated when an error name is referenced
// try wasm.populateErrorNameTable();
const objects = comp.objects;
// Positional arguments to the linker such as object files and static archives.
var positionals = std.ArrayList([]const u8).init(arena);
try positionals.ensureUnusedCapacity(objects.len);
for (objects) |object| {
positionals.appendAssumeCapacity(object.path);
}
for (comp.c_object_table.keys()) |c_object| {
try positionals.append(c_object.status.success.object_path);
}
if (comp.compiler_rt_lib) |lib| try positionals.append(lib.full_object_path);
if (comp.compiler_rt_obj) |obj| try positionals.append(obj.full_object_path);
try wasm.parseInputFiles(positionals.items);
if (wasm.zig_object_index != .null) {
try wasm.resolveSymbolsInObject(wasm.zig_object_index);
}
@@ -2752,73 +2541,11 @@ pub fn flushModule(wasm: *Wasm, arena: Allocator, prog_node: *std.Progress.Node)
try wasm.resolveLazySymbols();
try wasm.checkUndefinedSymbols();
// When we finish/error we reset the state of the linker
// So we can rebuild the binary file on each incremental update
defer wasm.resetState();
try wasm.setupInitFunctions();
try wasm.setupStart();
try wasm.markReferences();
// try wasm.setupErrorsLen();
try wasm.setupImports();
// if (comp.module) |mod| {
// var decl_it = wasm.decls.iterator();
// while (decl_it.next()) |entry| {
// const decl = mod.declPtr(entry.key_ptr.*);
// if (decl.isExtern(mod)) continue;
// const atom_index = entry.value_ptr.*;
// const atom = wasm.getAtomPtr(atom_index);
// if (decl.ty.zigTypeTag(mod) == .Fn) {
// try wasm.parseAtom(atom_index, .function);
// } else if (decl.getOwnedVariable(mod)) |variable| {
// if (variable.is_const) {
// try wasm.parseAtom(atom_index, .{ .data = .read_only });
// } else if (Value.fromInterned(variable.init).isUndefDeep(mod)) {
// // for safe build modes, we store the atom in the data segment,
// // whereas for unsafe build modes we store it in bss.
// const decl_namespace = mod.namespacePtr(decl.src_namespace);
// const optimize_mode = decl_namespace.file_scope.mod.optimize_mode;
// const is_initialized = switch (optimize_mode) {
// .Debug, .ReleaseSafe => true,
// .ReleaseFast, .ReleaseSmall => false,
// };
// try wasm.parseAtom(atom_index, .{ .data = if (is_initialized) .initialized else .uninitialized });
// } else {
// // when the decl is all zeroes, we store the atom in the bss segment,
// // in all other cases it will be in the data segment.
// const is_zeroes = for (atom.code.items) |byte| {
// if (byte != 0) break false;
// } else true;
// try wasm.parseAtom(atom_index, .{ .data = if (is_zeroes) .uninitialized else .initialized });
// }
// } else {
// try wasm.parseAtom(atom_index, .{ .data = .read_only });
// }
// // also parse atoms for a decl's locals
// for (atom.locals.items) |local_atom_index| {
// try wasm.parseAtom(local_atom_index, .{ .data = .read_only });
// }
// }
// // parse anonymous declarations
// for (wasm.anon_decls.keys(), wasm.anon_decls.values()) |decl_val, atom_index| {
// const ty = Type.fromInterned(mod.intern_pool.typeOf(decl_val));
// if (ty.zigTypeTag(mod) == .Fn) {
// try wasm.parseAtom(atom_index, .function);
// } else {
// try wasm.parseAtom(atom_index, .{ .data = .read_only });
// }
// }
// // also parse any backend-generated functions
// for (wasm.synthetic_functions.items) |atom_index| {
// try wasm.parseAtom(atom_index, .function);
// }
// if (wasm.dwarf) |*dwarf| {
// try dwarf.flushModule(comp.module.?);
// }
// }
try wasm.mergeSections();
try wasm.mergeTypes();
try wasm.allocateAtoms();
@@ -4032,7 +3759,7 @@ fn emitSymbolTable(wasm: *Wasm, binary_bytes: *std.ArrayList(u8), symbol_table:
try leb.writeULEB128(writer, @intFromEnum(symbol.tag));
try leb.writeULEB128(writer, symbol.flags);
const sym_name = if (wasm.export_names.get(sym_loc)) |exp_name| wasm.string_table.get(exp_name) else sym_loc.getName(wasm);
const sym_name = sym_loc.getName(wasm);
switch (symbol.tag) {
.data => {
try leb.writeULEB128(writer, @as(u32, @intCast(sym_name.len)));

14
src/link/Wasm/ZigObject.zig
View File

@@ -670,17 +670,15 @@ pub fn addOrUpdateImport(
if (type_index) |ty_index| {
const gop = try zig_object.imports.getOrPut(gpa, symbol_index);
const module_name = if (lib_name) |l_name| blk: {
break :blk l_name;
} else wasm_file.host_name;
const module_name = if (lib_name) |l_name| l_name else wasm_file.host_name;
if (!gop.found_existing) {
gop.value_ptr.* = .{
.module_name = try zig_object.string_table.insert(gpa, module_name),
.name = try zig_object.string_table.insert(gpa, name),
.kind = .{ .function = ty_index },
};
zig_object.imported_functions_count += 1;
}
gop.value_ptr.* = .{
.module_name = try zig_object.string_table.insert(gpa, module_name),
.name = try zig_object.string_table.insert(gpa, name),
.kind = .{ .function = ty_index },
};
sym.tag = .function;
} else {
sym.tag = .data;