From e17b2c9641fc5d3e73894633dffe93ecd91bf482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Sat, 5 Mar 2022 06:08:01 +0200 Subject: [PATCH] expose native shell sections --- README.md | 36 ++++++++++++++++++------------------ src/sections.zig | 36 ++++++++++++++++-------------------- src/shell.zig | 27 +++++++++++---------------- 3 files changed, 45 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 2014a16..926d165 100644 --- a/README.md +++ b/README.md @@ -349,23 +349,23 @@ Complete file structure Each section is padded to 64 bytes. ``` -STATUS SECTION SIZE DESCRIPTION -✅ Header 48 see "Turbonss header" section -✅ bdz_gid ? bdz(gid) -✅ bdz_groupname ? bdz(groupname) -✅ bdz_uid ? bdz(uid) -✅ bdz_username ? bdz(username) - idx_gid2group len(group)*29/8 bdz->offset Groups - idx_groupname2group len(group)*29/8 bdz->offset Groups - idx_uid2user len(user)*29/8 bdz->offset Users - idx_name2user len(user)*29/8 bdz->offset Users - idx_username2gids len(user)*29/8 bdz->offset UserGids -✅ shellIndex len(shells)*2 shell index array -✅ shellBlob <= 4032 shell data blob (max 63*64 bytes) -✅ groups ? packed Group entries (8b padding) -✅ users ? packed User entries (8b padding) - groupMembers ? per-group memberlist (no padding) - userGids ? per-user gidlist entries (8b padding) +SECTION SIZE DESCRIPTION +Header 48 see "Turbonss header" section +bdz_gid ? bdz(gid) +bdz_groupname ? bdz(groupname) +bdz_uid ? bdz(uid) +bdz_username ? bdz(username) +idx_gid2group len(group)*29/8 bdz->offset Groups +idx_groupname2group len(group)*29/8 bdz->offset Groups +idx_uid2user len(user)*29/8 bdz->offset Users +idx_name2user len(user)*29/8 bdz->offset Users +idx_username2gids len(user)*29/8 bdz->offset UserGids +shellIndex len(shells)*2 shell index array +shellBlob <= 4032 shell data blob (max 63*64 bytes) +groups ? packed Group entries (8b padding) +users ? packed User entries (8b padding) +groupMembers ? per-group memberlist (no padding) +userGids ? per-user gidlist entries (8b padding) ``` Section creation order: @@ -373,7 +373,7 @@ Section creation order: 1. ✅ `bdz_*`. No depdendencies. 1. ✅ `shellIndex`, `shellBlob`. No dependencies. 1. ✅ userGids. No dependencies. -1. Users. Requires `userGids`. +1. Users. Requires `userGids` and shell. 1. Groupmembers. Requires Users. 1. Groups. Requires Groupmembers. 1. `idx_*`. Requires offsets to Groups and Users. diff --git a/src/sections.zig b/src/sections.zig index d913956..1f148c5 100644 --- a/src/sections.zig +++ b/src/sections.zig @@ -20,6 +20,7 @@ const cmph = @import("cmph.zig"); const bdz = @import("bdz.zig"); const User = userImport.User; const Group = groupImport.Group; +const ShellSections = shellImport.ShellWriter.ShellSections; const Corpus = struct { arena: std.heap.ArenaAllocator, @@ -190,17 +191,6 @@ pub fn bdzUsername(allocator: Allocator, corpus: *const Corpus) cmph.Error![]con return try cmph.pack_str(allocator, corpus.usersMulti.items(.name)); } -pub const ShellSections = struct { - index: []const u8, - blob: []const u8, - - pub fn deinit(self: *ShellSections, allocator: Allocator) void { - allocator.free(self.index); - allocator.free(self.blob); - self.* = undefined; - } -}; - // TODO(motiejus) there are a few problems: // - memory management for shell sections is a mess. Make it easier by ... // - shell module should accept a list of shells and spit out two slices @@ -212,17 +202,10 @@ pub fn shellSections( corpus: *const Corpus, ) shellSectionsErr!ShellSections { var popcon = shellImport.ShellWriter.init(allocator); - defer popcon.deinit(); for (corpus.usersMulti.items(.shell)) |shell| { try popcon.put(shell); } - var sections = try popcon.toOwnedSections(shellImport.max_shells); - defer sections.deinit(); - - return ShellSections{ - .index = try allocator.dupe(u8, sections.sectionIndex()), - .blob = try allocator.dupe(u8, sections.sectionBlob()), - }; + return popcon.toOwnedSections(shellImport.max_shells); } pub const UserGids = struct { @@ -276,6 +259,19 @@ pub fn userGids(allocator: Allocator, corpus: *const Corpus) userGidsErr!UserGid }; } +pub const usersSectionErr = Allocator.Error || error{Overflow}; +pub fn usersSection( + allocator: Allocator, + corpus: *const Corpus, + gids: UserGids, + shells: ShellSections, +) usersSectionErr![]const u8 { + _ = allocator; + _ = corpus; + _ = gids; + _ = shells; +} + pub fn groupMembers(allocator: Allocator, corpus: *const Corpus) Allocator.Error!void { var buf: [compress.maxVarintLen64]u8 = undefined; var offsets = ArrayListUnmanaged(usize).initCapacity( @@ -447,7 +443,7 @@ test "test sections" { defer allocator.free(bdz_username); var shell_sections = try shellSections(allocator, &corpus); - defer shell_sections.deinit(allocator); + defer shell_sections.deinit(); var user_gids = try userGids(allocator, &corpus); defer user_gids.deinit(allocator); diff --git a/src/shell.zig b/src/shell.zig index da54250..186b036 100644 --- a/src/shell.zig +++ b/src/shell.zig @@ -54,7 +54,7 @@ pub const ShellWriter = struct { score: u64, }; - const ShellSections = struct { + pub const ShellSections = struct { index: BoundedArray(ShellIndex, max_shells), blob: BoundedArray(u8, max_shells * max_shell_len), indices: StringHashMap(u6), @@ -71,15 +71,16 @@ pub const ShellWriter = struct { .blob = try BoundedArray(u8, max_shells * max_shell_len).init(0), .indices = StringHashMap(u6).init(allocator), }; + errdefer self.indices.deinit(); var fullOffset: u12 = 0; var idx: u6 = 0; while (idx < shells.len) : (idx += 1) { - const len = @intCast(u6, shells.get(idx).len); + const len = try std.math.cast(u6, shells.get(idx).len); try self.blob.appendSlice(shells.get(idx)); const ourShell = self.blob.constSlice()[fullOffset .. fullOffset + len]; try self.indices.put(ourShell, idx); self.index.set(idx, ShellIndex{ - .offset = @intCast(u10, fullOffset >> 2), + .offset = try std.math.cast(u10, fullOffset >> 2), .len = len - 1, }); @@ -118,26 +119,23 @@ pub const ShellWriter = struct { pub fn deinit(self: *ShellWriter) void { var it = self.counts.keyIterator(); - while (it.next()) |key_ptr| { + while (it.next()) |key_ptr| self.counts.allocator.free(key_ptr.*); - } self.counts.deinit(); self.* = undefined; } pub fn put(self: *ShellWriter, shell: []const u8) !void { const res = try self.counts.getOrPutAdapted(shell, self.counts.ctx); - - if (res.found_existing) { - res.value_ptr.* += 1; - } else { + if (!res.found_existing) { res.key_ptr.* = try self.allocator.dupe(u8, shell); res.value_ptr.* = 1; + } else { + res.value_ptr.* += 1; } } - fn cmpShells(context: void, a: KV, b: KV) std.math.Order { - _ = context; + fn cmpShells(_: void, a: KV, b: KV) std.math.Order { return std.math.order(a.score, b.score); } @@ -161,10 +159,8 @@ pub const ShellWriter = struct { var topShells = try BoundedArray([]const u8, max_shells).init(total); var i: u32 = 0; - while (i < total) : (i += 1) { - const elem = deque.removeMax().shell; - topShells.set(i, elem); - } + while (i < total) : (i += 1) + topShells.set(i, deque.removeMax().shell); const result = ShellSections.init(self.allocator, topShells); const allocator = self.allocator; @@ -178,7 +174,6 @@ const testing = std.testing; test "basic shellpopcon" { var popcon = ShellWriter.init(testing.allocator); - defer popcon.deinit(); const bash = "/bin/bash"; // 9 chars const zsh = "/bin/zsh"; // 8 chars