expose native shell sections

README.md

@@ -349,23 +349,23 @@ Complete file structure
 Each section is padded to 64 bytes.
 
 ```
-STATUS  SECTION               SIZE             DESCRIPTION
+SECTION               SIZE             DESCRIPTION
-✅      Header                48               see "Turbonss header" section
+Header                48               see "Turbonss header" section
-✅      bdz_gid               ?                bdz(gid)
+bdz_gid               ?                bdz(gid)
-✅      bdz_groupname         ?                bdz(groupname)
+bdz_groupname         ?                bdz(groupname)
-✅      bdz_uid               ?                bdz(uid)
+bdz_uid               ?                bdz(uid)
-✅      bdz_username          ?                bdz(username)
+bdz_username          ?                bdz(username)
-        idx_gid2group         len(group)*29/8  bdz->offset Groups
+idx_gid2group         len(group)*29/8  bdz->offset Groups
-        idx_groupname2group   len(group)*29/8  bdz->offset Groups
+idx_groupname2group   len(group)*29/8  bdz->offset Groups
-        idx_uid2user          len(user)*29/8   bdz->offset Users
+idx_uid2user          len(user)*29/8   bdz->offset Users
-        idx_name2user         len(user)*29/8   bdz->offset Users
+idx_name2user         len(user)*29/8   bdz->offset Users
-        idx_username2gids     len(user)*29/8   bdz->offset UserGids
+idx_username2gids     len(user)*29/8   bdz->offset UserGids
-✅      shellIndex            len(shells)*2    shell index array
+shellIndex            len(shells)*2    shell index array
-✅      shellBlob             <= 4032          shell data blob (max 63*64 bytes)
+shellBlob             <= 4032          shell data blob (max 63*64 bytes)
-✅      groups                ?                packed Group entries (8b padding)
+groups                ?                packed Group entries (8b padding)
-✅      users                 ?                packed User entries (8b padding)
+users                 ?                packed User entries (8b padding)
-        groupMembers          ?                per-group memberlist (no padding)
+groupMembers          ?                per-group memberlist (no padding)
-        userGids              ?                per-user gidlist entries (8b padding)
+userGids              ?                per-user gidlist entries (8b padding)
 ```
 
 Section creation order:
@@ -373,7 +373,7 @@ Section creation order:
 1. ✅ `bdz_*`. No depdendencies.
 1. ✅ `shellIndex`, `shellBlob`. No dependencies.
 1. ✅ userGids. No dependencies.
-1. Users. Requires `userGids`.
+1. Users. Requires `userGids` and shell.
 1. Groupmembers. Requires Users.
 1. Groups. Requires Groupmembers.
 1. `idx_*`. Requires offsets to Groups and Users.

src/sections.zig

@@ -20,6 +20,7 @@ const cmph = @import("cmph.zig");
 const bdz = @import("bdz.zig");
 const User = userImport.User;
 const Group = groupImport.Group;
+const ShellSections = shellImport.ShellWriter.ShellSections;
 
 const Corpus = struct {
     arena: std.heap.ArenaAllocator,
@@ -190,17 +191,6 @@ pub fn bdzUsername(allocator: Allocator, corpus: *const Corpus) cmph.Error![]con
     return try cmph.pack_str(allocator, corpus.usersMulti.items(.name));
 }
 
-pub const ShellSections = struct {
-    index: []const u8,
-    blob: []const u8,
-
-    pub fn deinit(self: *ShellSections, allocator: Allocator) void {
-        allocator.free(self.index);
-        allocator.free(self.blob);
-        self.* = undefined;
-    }
-};
-
 // TODO(motiejus) there are a few problems:
 // - memory management for shell sections is a mess. Make it easier by ...
 // - shell module should accept a list of shells and spit out two slices
@@ -212,17 +202,10 @@ pub fn shellSections(
     corpus: *const Corpus,
 ) shellSectionsErr!ShellSections {
     var popcon = shellImport.ShellWriter.init(allocator);
-    defer popcon.deinit();
     for (corpus.usersMulti.items(.shell)) |shell| {
         try popcon.put(shell);
     }
-    var sections = try popcon.toOwnedSections(shellImport.max_shells);
+    return popcon.toOwnedSections(shellImport.max_shells);
-    defer sections.deinit();
-
-    return ShellSections{
-        .index = try allocator.dupe(u8, sections.sectionIndex()),
-        .blob = try allocator.dupe(u8, sections.sectionBlob()),
-    };
 }
 
 pub const UserGids = struct {
@@ -276,6 +259,19 @@ pub fn userGids(allocator: Allocator, corpus: *const Corpus) userGidsErr!UserGid
     };
 }
 
+pub const usersSectionErr = Allocator.Error || error{Overflow};
+pub fn usersSection(
+    allocator: Allocator,
+    corpus: *const Corpus,
+    gids: UserGids,
+    shells: ShellSections,
+) usersSectionErr![]const u8 {
+    _ = allocator;
+    _ = corpus;
+    _ = gids;
+    _ = shells;
+}
+
 pub fn groupMembers(allocator: Allocator, corpus: *const Corpus) Allocator.Error!void {
     var buf: [compress.maxVarintLen64]u8 = undefined;
     var offsets = ArrayListUnmanaged(usize).initCapacity(
@@ -447,7 +443,7 @@ test "test sections" {
     defer allocator.free(bdz_username);
 
     var shell_sections = try shellSections(allocator, &corpus);
-    defer shell_sections.deinit(allocator);
+    defer shell_sections.deinit();
 
     var user_gids = try userGids(allocator, &corpus);
     defer user_gids.deinit(allocator);

src/shell.zig

@@ -54,7 +54,7 @@ pub const ShellWriter = struct {
         score: u64,
     };
 
-    const ShellSections = struct {
+    pub const ShellSections = struct {
         index: BoundedArray(ShellIndex, max_shells),
         blob: BoundedArray(u8, max_shells * max_shell_len),
         indices: StringHashMap(u6),
@@ -71,15 +71,16 @@ pub const ShellWriter = struct {
                 .blob = try BoundedArray(u8, max_shells * max_shell_len).init(0),
                 .indices = StringHashMap(u6).init(allocator),
             };
+            errdefer self.indices.deinit();
             var fullOffset: u12 = 0;
             var idx: u6 = 0;
             while (idx < shells.len) : (idx += 1) {
-                const len = @intCast(u6, shells.get(idx).len);
+                const len = try std.math.cast(u6, shells.get(idx).len);
                 try self.blob.appendSlice(shells.get(idx));
                 const ourShell = self.blob.constSlice()[fullOffset .. fullOffset + len];
                 try self.indices.put(ourShell, idx);
                 self.index.set(idx, ShellIndex{
-                    .offset = @intCast(u10, fullOffset >> 2),
+                    .offset = try std.math.cast(u10, fullOffset >> 2),
                     .len = len - 1,
                 });
 
@@ -118,26 +119,23 @@ pub const ShellWriter = struct {
 
     pub fn deinit(self: *ShellWriter) void {
         var it = self.counts.keyIterator();
-        while (it.next()) |key_ptr| {
+        while (it.next()) |key_ptr|
             self.counts.allocator.free(key_ptr.*);
-        }
         self.counts.deinit();
         self.* = undefined;
     }
 
     pub fn put(self: *ShellWriter, shell: []const u8) !void {
         const res = try self.counts.getOrPutAdapted(shell, self.counts.ctx);
-
+        if (!res.found_existing) {
-        if (res.found_existing) {
-            res.value_ptr.* += 1;
-        } else {
             res.key_ptr.* = try self.allocator.dupe(u8, shell);
             res.value_ptr.* = 1;
+        } else {
+            res.value_ptr.* += 1;
         }
     }
 
-    fn cmpShells(context: void, a: KV, b: KV) std.math.Order {
+    fn cmpShells(_: void, a: KV, b: KV) std.math.Order {
-        _ = context;
         return std.math.order(a.score, b.score);
     }
 
@@ -161,10 +159,8 @@ pub const ShellWriter = struct {
         var topShells = try BoundedArray([]const u8, max_shells).init(total);
 
         var i: u32 = 0;
-        while (i < total) : (i += 1) {
+        while (i < total) : (i += 1)
-            const elem = deque.removeMax().shell;
+            topShells.set(i, deque.removeMax().shell);
-            topShells.set(i, elem);
-        }
 
         const result = ShellSections.init(self.allocator, topShells);
         const allocator = self.allocator;
@@ -178,7 +174,6 @@ const testing = std.testing;
 
 test "basic shellpopcon" {
     var popcon = ShellWriter.init(testing.allocator);
-    defer popcon.deinit();
 
     const bash = "/bin/bash"; // 9 chars
     const zsh = "/bin/zsh"; // 8 chars