diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 6539946ca1..e151846e5a 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -105,7 +105,7 @@ atoms: std.ArrayListUnmanaged(Atom) = .{}, /// /// value assigned to label `foo` is an unnamed constant belonging/associated /// with `Decl` `main`, and lives as long as that `Decl`. -unnamed_const_atoms: UnnamedConstTable = .{}, +unnamed_consts: UnnamedConstTable = .{}, /// A table of relocations indexed by the owning them `TextBlock`. relocs: RelocTable = .{}, @@ -251,11 +251,11 @@ pub fn deinit(self: *Elf) void { self.lazy_syms.deinit(gpa); { - var it = self.unnamed_const_atoms.valueIterator(); - while (it.next()) |atoms| { - atoms.deinit(gpa); + var it = self.unnamed_consts.valueIterator(); + while (it.next()) |syms| { + syms.deinit(gpa); } - self.unnamed_const_atoms.deinit(gpa); + self.unnamed_consts.deinit(gpa); } { @@ -279,7 +279,7 @@ pub fn getDeclVAddr(self: *Elf, decl_index: Module.Decl.Index, reloc_info: link. const vaddr = this_sym.value; const parent_atom_index = self.symbol(reloc_info.parent_atom_index).atom_index; try Atom.addRelocation(self, parent_atom_index, .{ - .target = this_sym, + .target = this_sym_index, .offset = reloc_info.offset, .addend = reloc_info.addend, .prev_vaddr = vaddr, @@ -830,6 +830,12 @@ pub fn populateMissingMetadata(self: *Elf) !void { try self.base.file.?.pwriteAll(&[_]u8{0}, max_file_offset); } + + if (self.zig_module_index == null) { + const index = @as(File.Index, @intCast(try self.files.addOne(gpa))); + self.files.set(index, .{ .zig_module = .{ .index = index } }); + self.zig_module_index = index; + } } pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void { @@ -967,12 +973,6 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node const module = self.base.options.module orelse return error.LinkingWithoutZigSourceUnimplemented; _ = module; - self.zig_module_index = blk: { - const index = @as(File.Index, @intCast(try self.files.addOne(gpa))); - self.files.set(index, .{ .zig_module = .{ .index = index } }); - break :blk index; - }; - self.linker_defined_index = blk: { const index = @as(File.Index, @intCast(try self.files.addOne(gpa))); self.files.set(index, .{ .linker_defined = .{ .index = index } }); @@ -980,6 +980,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node }; std.debug.print("{}\n", .{self.dumpState()}); + return error.FlushFailure; // if (self.lazy_syms.getPtr(.none)) |metadata| { // // Most lazy symbols can be updated on first use, but @@ -2078,80 +2079,23 @@ fn writeElfHeader(self: *Elf) !void { try self.base.file.?.pwriteAll(hdr_buf[0..index], 0); } -fn freeAtom(self: *Elf, atom_index: Atom.Index) void { - const atom_ptr = self.atom(atom_index); - log.debug("freeAtom {d} ({s})", .{ atom_index, atom_ptr.name(self) }); - - Atom.freeRelocations(self, atom_index); - - const gpa = self.base.allocator; - const shndx = atom_ptr.symbol(self).st_shndx; - const free_list = &self.sections.items(.free_list)[shndx]; - var already_have_free_list_node = false; - { - var i: usize = 0; - // TODO turn free_list into a hash map - while (i < free_list.items.len) { - if (free_list.items[i] == atom_index) { - _ = free_list.swapRemove(i); - continue; - } - if (free_list.items[i] == atom_ptr.prev_index) { - already_have_free_list_node = true; - } - i += 1; - } - } - - const maybe_last_atom_index = &self.sections.items(.last_atom_index)[shndx]; - if (maybe_last_atom_index.*) |last_atom_index| { - if (last_atom_index == atom_index) { - if (atom_ptr.prev_index) |prev_index| { - // TODO shrink the section size here - maybe_last_atom_index.* = prev_index; - } else { - maybe_last_atom_index.* = null; - } - } - } - - if (atom_ptr.prev_index) |prev_index| { - const prev = self.atom(prev_index); - prev.next_index = atom_ptr.next_index; - - if (!already_have_free_list_node and prev.*.freeListEligible(self)) { - // The free list is heuristics, it doesn't have to be perfect, so we can - // ignore the OOM here. - free_list.append(gpa, prev_index) catch {}; - } - } else { - atom_ptr.prev_index = null; - } - - if (atom_ptr.next_index) |next_index| { - self.atom(next_index).prev_index = atom_ptr.prev_index; - } else { - atom_ptr.next_index = null; - } - - // Appending to free lists is allowed to fail because the free lists are heuristics based anyway. - const sym_index = atom_ptr.symbolIndex().?; - - log.debug("adding %{d} to local symbols free list", .{sym_index}); - self.symbols_free_list.append(gpa, sym_index) catch {}; - self.symbols.items[sym_index] = .{}; - atom_ptr.sym_index = 0; - self.got_table.freeEntry(gpa, sym_index); -} - fn freeUnnamedConsts(self: *Elf, decl_index: Module.Decl.Index) void { - const unnamed_consts = self.unnamed_const_atoms.getPtr(decl_index) orelse return; - for (unnamed_consts.items) |atom_index| { - self.freeAtom(atom_index); + const unnamed_consts = self.unnamed_consts.getPtr(decl_index) orelse return; + for (unnamed_consts.items) |sym_index| { + self.freeDeclMetadata(sym_index); } unnamed_consts.clearAndFree(self.base.allocator); } +fn freeDeclMetadata(self: *Elf, sym_index: Symbol.Index) void { + const sym = self.symbol(sym_index); + sym.atom(self).?.free(self); + log.debug("adding %{d} to local symbols free list", .{sym_index}); + self.symbols_free_list.append(self.base.allocator, sym_index) catch {}; + self.symbols.items[sym_index] = .{}; + self.got_table.freeEntry(self.base.allocator, sym_index); +} + pub fn freeDecl(self: *Elf, decl_index: Module.Decl.Index) void { if (self.llvm_object) |llvm_object| return llvm_object.freeDecl(decl_index); @@ -2162,7 +2106,8 @@ pub fn freeDecl(self: *Elf, decl_index: Module.Decl.Index) void { if (self.decls.fetchRemove(decl_index)) |const_kv| { var kv = const_kv; - self.freeAtom(kv.value.atom); + const sym_index = kv.value.symbol_index; + self.freeDeclMetadata(sym_index); self.freeUnnamedConsts(decl_index); kv.value.exports.deinit(self.base.allocator); } @@ -2196,7 +2141,7 @@ pub fn getOrCreateMetadataForLazySymbol(self: *Elf, sym: link.File.LazySymbol) ! .code => self.text_section_index.?, .const_data => self.rodata_section_index.?, }, self), - .pending_flush => return metadata.atom.*, + .pending_flush => return metadata.symbol_index.*, .flushed => {}, } metadata.state.* = .pending_flush; @@ -2271,6 +2216,7 @@ fn updateDeclCode( esym.st_size = code.len; const old_size = atom_ptr.size; + const old_vaddr = atom_ptr.value; atom_ptr.alignment = math.log2_int(u64, required_alignment); atom_ptr.size = code.len; @@ -2278,11 +2224,11 @@ fn updateDeclCode( const capacity = atom_ptr.capacity(self); const need_realloc = code.len > capacity or !mem.isAlignedGeneric(u64, sym.value, required_alignment); if (need_realloc) { - const vaddr = try atom_ptr.grow(self); - log.debug("growing {s} from 0x{x} to 0x{x}", .{ decl_name, sym.value, vaddr }); - if (vaddr != sym.value) { - sym.value = vaddr; - esym.st_value = vaddr; + try atom_ptr.grow(self); + log.debug("growing {s} from 0x{x} to 0x{x}", .{ decl_name, old_vaddr, atom_ptr.value }); + if (old_vaddr != atom_ptr.value) { + sym.value = atom_ptr.value; + esym.st_value = atom_ptr.value; log.debug(" (writing new offset table entry)", .{}); const got_entry_index = self.got_table.lookup.get(sym_index).?; @@ -2293,12 +2239,16 @@ fn updateDeclCode( atom_ptr.shrink(self); } } else { - const vaddr = try atom_ptr.allocate(self); - errdefer self.freeAtom(atom_ptr); - log.debug("allocated text block for {s} at 0x{x}", .{ decl_name, vaddr }); + try atom_ptr.allocate(self); + errdefer self.freeDeclMetadata(sym_index); + log.debug("allocated atom for {s} at 0x{x} to 0x{x}", .{ + decl_name, + atom_ptr.value, + atom_ptr.value + atom_ptr.size, + }); - sym.value = vaddr; - esym.st_value = vaddr; + sym.value = atom_ptr.value; + esym.st_value = atom_ptr.value; const got_entry_index = try sym.getOrCreateOffsetTableEntry(self); try self.writeOffsetTableEntry(got_entry_index); @@ -2518,17 +2468,23 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol. const atom_ptr = local_sym.atom(self).?; atom_ptr.alignment = math.log2_int(u64, required_alignment); atom_ptr.size = code.len; - const vaddr = try atom_ptr.allocate(self); - errdefer self.freeAtom(atom_ptr); - log.debug("allocated text block for {s} at 0x{x}", .{ name, vaddr }); - local_sym.value = vaddr; - local_esym.st_value = vaddr; + try atom_ptr.allocate(self); + errdefer self.freeDeclMetadata(symbol_index); + + log.debug("allocated atom for {s} at 0x{x} to 0x{x}", .{ + name, + atom_ptr.value, + atom_ptr.value + atom_ptr.size, + }); + + local_sym.value = atom_ptr.value; + local_esym.st_value = atom_ptr.value; const got_entry_index = try local_sym.getOrCreateOffsetTableEntry(self); try self.writeOffsetTableEntry(got_entry_index); - const section_offset = vaddr - self.program_headers.items[phdr_index].p_vaddr; + const section_offset = atom_ptr.value - self.program_headers.items[phdr_index].p_vaddr; const file_offset = self.sections.items(.shdr)[local_sym.output_section_index].sh_offset + section_offset; try self.base.file.?.pwriteAll(code, file_offset); } @@ -2540,7 +2496,7 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module defer code_buffer.deinit(); const mod = self.base.options.module.?; - const gop = try self.unnamed_const_atoms.getOrPut(gpa, decl_index); + const gop = try self.unnamed_consts.getOrPut(gpa, decl_index); if (!gop.found_existing) { gop.value_ptr.* = .{}; } @@ -2586,17 +2542,18 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module const atom_ptr = local_sym.atom(self).?; atom_ptr.alignment = math.log2_int(u64, required_alignment); atom_ptr.size = code.len; - const vaddr = try atom_ptr.allocateAtom(self); - errdefer self.freeAtom(atom_ptr); - log.debug("allocated text block for {s} at 0x{x}", .{ name, local_sym.st_value }); + try atom_ptr.allocate(self); + errdefer self.freeDeclMetadata(sym_index); - local_sym.value = vaddr; - local_esym.st_value = vaddr; + log.debug("allocated atom for {s} at 0x{x} to 0x{x}", .{ name, atom_ptr.value, atom_ptr.value + atom_ptr.size }); + + local_sym.value = atom_ptr.value; + local_esym.st_value = atom_ptr.value; try unnamed_consts.append(gpa, atom_ptr.atom_index); - const section_offset = local_sym.value - self.program_headers.items[phdr_index].p_vaddr; + const section_offset = atom_ptr.value - self.program_headers.items[phdr_index].p_vaddr; const file_offset = self.sections.items(.shdr)[shdr_index].sh_offset + section_offset; try self.base.file.?.pwriteAll(code, file_offset); @@ -2624,7 +2581,7 @@ pub fn updateDeclExports( const decl = mod.declPtr(decl_index); const decl_sym_index = try self.getOrCreateMetadataForDecl(decl_index); const decl_sym = self.symbol(decl_sym_index); - const decl_esym = symbol.sourceSymbol(self); + const decl_esym = decl_sym.sourceSymbol(self); const decl_metadata = self.decls.getPtr(decl_index).?; for (exports) |exp| { @@ -2658,7 +2615,7 @@ pub fn updateDeclExports( continue; }, }; - const stt_bits: u8 = @as(u4, @truncate(decl_sym.st_info)); + const stt_bits: u8 = @as(u4, @truncate(decl_esym.st_info)); const sym_index = if (decl_metadata.@"export"(self, exp_name)) |exp_index| exp_index.* else blk: { const zig_module = self.file(self.zig_module_index.?).?.zig_module; @@ -3220,7 +3177,7 @@ pub fn addSymbol(self: *Elf) !Symbol.Index { break :blk index; } }; - self.symbols.items[index] = .{ .symbol_index = index }; + self.symbols.items[index] = .{ .index = index }; return index; } diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index faebd0b6fa..8eb9f1af0b 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -79,7 +79,7 @@ pub fn freeRelocations(elf_file: *Elf, atom_index: Index) void { if (removed_relocs) |*relocs| relocs.value.deinit(elf_file.base.allocator); } -pub fn allocate(self: *Atom, elf_file: *Elf) !u64 { +pub fn allocate(self: *Atom, elf_file: *Elf) !void { const phdr_index = elf_file.sections.items(.phdr_index)[self.output_section_index]; const phdr = &elf_file.program_headers.items[phdr_index]; const shdr = &elf_file.sections.items(.shdr)[self.output_section_index]; @@ -98,7 +98,7 @@ pub fn allocate(self: *Atom, elf_file: *Elf) !u64 { // First we look for an appropriately sized free list node. // The list is unordered. We'll just take the first thing that works. - const vaddr = blk: { + self.value = blk: { var i: usize = if (elf_file.base.child_pid == null) 0 else free_list.items.len; while (i < free_list.items.len) { const big_atom_index = free_list.items[i]; @@ -152,7 +152,7 @@ pub fn allocate(self: *Atom, elf_file: *Elf) !u64 { else true; if (expand_section) { - const needed_size = (vaddr + self.size) - phdr.p_vaddr; + const needed_size = (self.value + self.size) - phdr.p_vaddr; try elf_file.growAllocSection(self.output_section_index, needed_size); maybe_last_atom_index.* = self.atom_index; @@ -193,7 +193,6 @@ pub fn allocate(self: *Atom, elf_file: *Elf) !u64 { if (free_list_removal) |i| { _ = free_list.swapRemove(i); } - return vaddr; } pub fn shrink(self: *Atom, elf_file: *Elf) void { @@ -201,12 +200,69 @@ pub fn shrink(self: *Atom, elf_file: *Elf) void { _ = elf_file; } -pub fn grow(self: *Atom, elf_file: *Elf) !u64 { +pub fn grow(self: *Atom, elf_file: *Elf) !void { const alignment = try std.math.powi(u64, 2, self.alignment); const align_ok = std.mem.alignBackward(u64, self.value, alignment) == self.value; const need_realloc = !align_ok or self.size > self.capacity(elf_file); - if (!need_realloc) return self.value; - return self.allocate(elf_file); + if (need_realloc) try self.allocate(elf_file); +} + +pub fn free(self: *Atom, elf_file: *Elf) void { + log.debug("freeAtom {d} ({s})", .{ self.atom_index, self.name(elf_file) }); + + Atom.freeRelocations(elf_file, self.atom_index); + + const gpa = elf_file.base.allocator; + const shndx = self.output_section_index; + const free_list = &elf_file.sections.items(.free_list)[shndx]; + var already_have_free_list_node = false; + { + var i: usize = 0; + // TODO turn free_list into a hash map + while (i < free_list.items.len) { + if (free_list.items[i] == self.atom_index) { + _ = free_list.swapRemove(i); + continue; + } + if (free_list.items[i] == self.prev_index) { + already_have_free_list_node = true; + } + i += 1; + } + } + + const maybe_last_atom_index = &elf_file.sections.items(.last_atom_index)[shndx]; + if (maybe_last_atom_index.*) |last_atom_index| { + if (last_atom_index == self.atom_index) { + if (self.prev_index) |prev_index| { + // TODO shrink the section size here + maybe_last_atom_index.* = prev_index; + } else { + maybe_last_atom_index.* = null; + } + } + } + + if (self.prev_index) |prev_index| { + const prev = elf_file.atom(prev_index); + prev.next_index = self.next_index; + + if (!already_have_free_list_node and prev.*.freeListEligible(elf_file)) { + // The free list is heuristics, it doesn't have to be perfect, so we can + // ignore the OOM here. + free_list.append(gpa, prev_index) catch {}; + } + } else { + self.prev_index = null; + } + + if (self.next_index) |next_index| { + elf_file.atom(next_index).prev_index = self.prev_index; + } else { + self.next_index = null; + } + + self.* = .{}; } pub const Index = u32; @@ -221,6 +277,7 @@ pub const Reloc = struct { const std = @import("std"); const assert = std.debug.assert; const elf = std.elf; +const log = std.log.scoped(.link); const Atom = @This(); const Elf = @import("../Elf.zig"); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 79934840c2..0418aa59db 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -1,5 +1,7 @@ //! Represents a defined symbol. +index: Index = 0, + /// Allocated address value of this symbol. value: u64 = 0, @@ -18,8 +20,8 @@ atom_index: Atom.Index = 0, output_section_index: u16 = 0, /// Index of the source symbol this symbol references. -/// Use `getSourceSymbol` to pull the source symbol from the relevant file. -symbol_index: Index = 0, +/// Use `sourceSymbol` to pull the source symbol from the relevant file. +esym_index: Index = 0, /// Index of the source version symbol this symbol references if any. /// If the symbol is unversioned it will have either VER_NDX_LOCAL or VER_NDX_GLOBAL. @@ -64,7 +66,11 @@ pub fn file(symbol: Symbol, elf_file: *Elf) ?File { } pub fn sourceSymbol(symbol: Symbol, elf_file: *Elf) *elf.Elf64_Sym { - return symbol.file(elf_file).?.sourceSymbol(symbol.symbol_index); + const file_ptr = symbol.file(elf_file).?; + switch (file_ptr) { + .zig_module => return file_ptr.zig_module.sourceSymbol(symbol.index, elf_file), + .linker_defined => return file_ptr.linker_defined.sourceSymbol(symbol.esym_index), + } } pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { @@ -80,14 +86,14 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { /// If entry already exists, returns index to it. /// Otherwise, creates a new entry in the Global Offset Table for this Symbol. pub fn getOrCreateOffsetTableEntry(self: Symbol, elf_file: *Elf) !Symbol.Index { - if (elf_file.got_table.lookup.get(self.symbol_index)) |index| return index; - const index = try elf_file.got_table.allocateEntry(elf_file.base.allocator, self.symbol_index); + if (elf_file.got_table.lookup.get(self.index)) |index| return index; + const index = try elf_file.got_table.allocateEntry(elf_file.base.allocator, self.index); elf_file.got_table_count_dirty = true; return index; } pub fn getOffsetTableAddress(self: Symbol, elf_file: *Elf) u64 { - const got_entry_index = elf_file.got_table.lookup.get(self.symbol_index).?; + const got_entry_index = elf_file.got_table.lookup.get(self.index).?; const target = elf_file.base.options.target; const ptr_bits = target.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); @@ -263,7 +269,7 @@ fn format2( _ = options; _ = unused_fmt_string; const symbol = ctx.symbol; - try writer.print("%{d} : {s} : @{x}", .{ symbol.symbol_index, symbol.fmtName(ctx.elf_file), symbol.value }); + try writer.print("%{d} : {s} : @{x}", .{ symbol.index, symbol.fmtName(ctx.elf_file), symbol.value }); if (symbol.file(ctx.elf_file)) |file_ptr| { if (symbol.isAbs(ctx.elf_file)) { if (symbol.sourceSymbol(ctx.elf_file).st_shndx == elf.SHN_UNDEF) { diff --git a/src/link/Elf/ZigModule.zig b/src/link/Elf/ZigModule.zig index ebfcb6c9e4..6cf918c8a1 100644 --- a/src/link/Elf/ZigModule.zig +++ b/src/link/Elf/ZigModule.zig @@ -33,6 +33,7 @@ pub fn createAtom(self: *ZigModule, output_section_index: u16, elf_file: *Elf) ! symbol_ptr.file_index = self.index; symbol_ptr.atom_index = atom_index; symbol_ptr.output_section_index = output_section_index; + symbol_ptr.esym_index = @as(Symbol.Index, @intCast(self.elf_local_symbols.items.len)); const local_esym = try self.elf_local_symbols.addOne(gpa); local_esym.* = .{ @@ -55,6 +56,7 @@ pub fn addGlobal(self: *ZigModule, name: [:0]const u8, elf_file: *Elf) !Symbol.I try self.elf_global_symbols.ensureUnusedCapacity(gpa, 1); try self.global_symbols.ensureUnusedCapacity(gpa, 1); const off = try elf_file.strtab.insert(gpa, name); + const esym_index = @as(Symbol.Index, @intCast(self.elf_global_symbols.items.len)); self.elf_global_symbols.appendAssumeCapacity(.{ .st_name = off, .st_info = elf.STB_GLOBAL << 4, @@ -64,14 +66,18 @@ pub fn addGlobal(self: *ZigModule, name: [:0]const u8, elf_file: *Elf) !Symbol.I .st_size = 0, }); const gop = try elf_file.getOrPutGlobal(off); + const sym = elf_file.symbol(gop.index); + sym.file_index = self.index; + sym.esym_index = esym_index; self.global_symbols.putAssumeCapacityNoClobber(gop.index, {}); return gop.index; } -pub fn sourceSymbol(self: *ZigModule, symbol_index: Symbol.Index) *elf.Elf64_Sym { - if (self.local_symbols.get(symbol_index)) |_| return &self.elf_local_symbols.items[symbol_index]; +pub fn sourceSymbol(self: *ZigModule, symbol_index: Symbol.Index, elf_file: *Elf) *elf.Elf64_Sym { + const sym = elf_file.symbol(symbol_index); + if (self.local_symbols.get(symbol_index)) |_| return &self.elf_local_symbols.items[sym.esym_index]; assert(self.global_symbols.get(symbol_index) != null); - return &self.elf_global_symbols.items[symbol_index]; + return &self.elf_global_symbols.items[sym.esym_index]; } pub fn locals(self: *ZigModule) []const Symbol.Index { diff --git a/src/link/Elf/file.zig b/src/link/Elf/file.zig index 45a063cda5..82d2d89c29 100644 --- a/src/link/Elf/file.zig +++ b/src/link/Elf/file.zig @@ -10,12 +10,6 @@ pub const File = union(enum) { }; } - pub fn sourceSymbol(file: File, symbol_index: Symbol.Index) *elf.Elf64_Sym { - return switch (file) { - inline else => |x| x.sourceSymbol(symbol_index), - }; - } - pub fn fmtPath(file: File) std.fmt.Formatter(formatPath) { return .{ .data = file }; }