add usergids

README.md

@@ -372,11 +372,10 @@ Section creation order:
 
 1. ✅ `bdz_*`. No depdendencies.
 1. ✅ `shellIndex`, `shellBlob`. No dependencies.
-1. UserGids. No dependencies.
+1. ✅ userGids. No dependencies.
-1. Users, but without `additional_gids_offset`. No dependencies.
+1. Users. Requires `userGids`.
-1. Groupmembers. Depends on Users, ex. `additional_gids_offset`.
+1. Groupmembers. Requires Users.
 1. Groups. Requires Groupmembers.
-1. Mutate `Users.additional_gids_offset`. Requires Groupmembers and ShellIndex.
 1. `idx_*`. Requires offsets to Groups and Users.
 1. Header.
 

src/compress.zig

@@ -5,6 +5,9 @@
 // golang's varint implementation.
 const std = @import("std");
 
+const ArrayList = std.ArrayList;
+const Allocator = std.mem.Allocator;
+
 // compresses a strictly incrementing sorted slice of integers using delta
 // compression. Compression is in-place.
 pub fn deltaCompress(comptime T: type, elems: []T) error{NotSorted}!void {
@@ -48,7 +51,7 @@ test "delta compress/decompress" {
         .{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } },
     };
     for (tests) |t| {
-        var arr = try std.ArrayList(u8).initCapacity(
+        var arr = try ArrayList(u8).initCapacity(
             testing.allocator,
             t.input.len,
         );
@@ -69,7 +72,7 @@ test "delta compression negative tests" {
         &[_]u8{ 0, 1, 1 },
         &[_]u8{ 0, 1, 2, 1 },
     }) |t| {
-        var arr = try std.ArrayList(u8).initCapacity(testing.allocator, t.len);
+        var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
         defer arr.deinit();
         try arr.appendSlice(t);
         try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items));
@@ -81,7 +84,7 @@ test "delta decompress overflow" {
         &[_]u8{ 255, 0 },
         &[_]u8{ 0, 128, 127 },
     }) |t| {
-        var arr = try std.ArrayList(u8).initCapacity(testing.allocator, t.len);
+        var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
         defer arr.deinit();
         try arr.appendSlice(t);
         try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items));
@@ -142,24 +145,31 @@ pub fn putUvarint(buf: []u8, x: u64) usize {
     return i + 1;
 }
 
-test "uvarint" {
+pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void {
-    const uvarint_tests = [_]u64{
+    var buf: [maxVarintLen64]u8 = undefined;
-        0,
+    const n = putUvarint(&buf, x);
-        1,
+    try arr.appendSlice(buf[0..n]);
-        2,
+}
-        10,
+
-        20,
+const uvarint_tests = [_]u64{
-        63,
+    0,
-        64,
+    1,
-        65,
+    2,
-        127,
+    10,
-        128,
+    20,
-        129,
+    63,
-        255,
+    64,
-        256,
+    65,
-        257,
+    127,
-        1 << 63 - 1,
+    128,
-    };
+    129,
+    255,
+    256,
+    257,
+    1 << 63 - 1,
+};
+
+test "putUvarint/uvarint" {
     for (uvarint_tests) |x| {
         var buf: [maxVarintLen64]u8 = undefined;
         const n = putUvarint(buf[0..], x);
@@ -170,6 +180,18 @@ test "uvarint" {
     }
 }
 
+test "appendUvarint" {
+    for (uvarint_tests) |x| {
+        var buf = ArrayList(u8).init(testing.allocator);
+        defer buf.deinit();
+
+        try appendUvarint(&buf, x);
+        const got = try uvarint(buf.items);
+
+        try testing.expectEqual(x, got.value);
+    }
+}
+
 test "overflow" {
     for ([_][]const u8{
         &[_]u8{ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2 },

src/sections.zig

@@ -108,6 +108,7 @@ const Corpus = struct {
         var username2groups = StringHashMap(
             ArrayListUnmanaged(*const Group),
         ).init(baseAllocator);
+
         for (groups) |*group| {
             var members = try allocator.alloc(*const User, group.members.count());
             members.len = 0;
@@ -143,9 +144,11 @@ const Corpus = struct {
             sort.sort(*const Group, userGroups.items, {}, cmpGroupPtr);
 
         var username2groups_final = StringHashMap([]*const Group).init(allocator);
-        for (users) |user| {
+        var it = username2groups.iterator();
-            const userGroups = username2groups.get(user.name).?.toOwnedSlice(allocator);
+        while (it.next()) |elem| {
-            try username2groups_final.put(user.name, userGroups);
+            const username = elem.key_ptr.*;
+            const usergroups = elem.value_ptr.*.toOwnedSlice(allocator);
+            try username2groups_final.put(username, usergroups);
         }
         username2groups.deinit();
 
@@ -228,6 +231,53 @@ pub const Sections = struct {
         };
     }
 
+    pub const UserGids = struct {
+        // username -> offset in blob
+        name2offset: StringHashMap(u32),
+        // compressed user gids blob. A blob contains N <= users.len items,
+        // an item is:
+        //   len: varint
+        //   gid: [varint]varint,
+        // ... and the gid list is delta-compressed.
+        blob: []u8,
+    };
+
+    const userGidsErr = Allocator.Error || error{Overflow};
+    pub fn userGids(self: *const Sections) userGidsErr!UserGids {
+        var arena = std.heap.ArenaAllocator.init(self.allocator);
+        defer arena.deinit();
+        var blob = ArrayList(u8).init(self.allocator);
+
+        var name2offset = StringHashMap(u32).init(self.allocator);
+        for (self.corpus.users) |user| {
+            const usergroups_maybe = self.corpus.username2groups.get(user.name);
+            if (usergroups_maybe == null)
+                continue;
+            const usergroups = usergroups_maybe.?;
+
+            try name2offset.putNoClobber(user.name, try math.cast(u32, blob.items.len));
+            var userBlob = try ArrayList(u8).initCapacity(arena.allocator(), usergroups.len * 2);
+            var deltaCompressedGids = try arena.allocator().alloc(u32, usergroups.len);
+            deltaCompressedGids.len = usergroups.len;
+            for (usergroups) |group, i| {
+                deltaCompressedGids[i] = group.gid;
+            }
+            compress.deltaCompress(u32, deltaCompressedGids) catch |err| switch (err) {
+                error.NotSorted => unreachable,
+            };
+            try compress.appendUvarint(&userBlob, usergroups.len);
+            for (deltaCompressedGids) |gid| {
+                try compress.appendUvarint(&userBlob, gid);
+            }
+            try blob.appendSlice(userBlob.toOwnedSlice());
+        }
+
+        return UserGids{
+            .name2offset = name2offset,
+            .blob = blob.toOwnedSlice(),
+        };
+    }
+
     pub fn groupMembers(self: *const Sections) Allocator.Error!GroupMembers {
         var buf: [compress.maxVarintLen64]u8 = undefined;
         var offsets = ArrayListUnmanaged(usize).initCapacity(
@@ -306,6 +356,13 @@ fn testCorpus(allocator: Allocator) !Corpus {
         .gecos = "",
         .home = "/",
         .shell = "/",
+    }, User{
+        .uid = 65534,
+        .gid = 65534,
+        .name = "nobody",
+        .gecos = "nobody",
+        .home = "/nonexistent",
+        .shell = "/usr/sbin/nologin",
     } };
 
     var members1 = try groupImport.someMembers(
@@ -348,8 +405,9 @@ test "test corpus" {
     defer corpus.deinit();
 
     try testing.expectEqualStrings(corpus.users[0].name, "Name" ** 8);
-    try testing.expectEqualStrings(corpus.users[1].name, "svc-bar");
+    try testing.expectEqualStrings(corpus.users[1].name, "nobody");
-    try testing.expectEqualStrings(corpus.users[2].name, "vidmantas");
+    try testing.expectEqualStrings(corpus.users[2].name, "svc-bar");
+    try testing.expectEqualStrings(corpus.users[3].name, "vidmantas");
 
     try testing.expectEqual(corpus.name2user.get("404"), null);
     try testing.expectEqual(corpus.name2user.get("vidmantas").?.uid, 128);
@@ -370,7 +428,8 @@ test "test corpus" {
     try testing.expectEqual(groupsOfVidmantas[0].gid, 0);
     try testing.expectEqual(groupsOfVidmantas[1].gid, 128);
     try testing.expectEqual(groupsOfVidmantas[2].gid, 9999);
-    try testing.expectEqual(corpus.username2groups.get("404"), null);
+    try testing.expectEqual(corpus.username2groups.get("nobody"), null);
+    try testing.expectEqual(corpus.username2groups.get("doesnotexist"), null);
 }
 
 test "test sections" {
@@ -392,9 +451,13 @@ test "test sections" {
     const bdz_username = try sections.bdzUsername();
     defer allocator.free(bdz_username);
 
-    const shellSections = try sections.shellSections();
+    const shell_sections = try sections.shellSections();
-    defer allocator.free(shellSections.index);
+    defer allocator.free(shell_sections.index);
-    defer allocator.free(shellSections.blob);
+    defer allocator.free(shell_sections.blob);
+
+    var user_gids = try sections.userGids();
+    defer user_gids.name2offset.deinit();
+    defer allocator.free(user_gids.blob);
 }
 
 test "pack gids" {

src/user.zig

@@ -31,7 +31,7 @@ pub const User = struct {
         const stringdata = try allocator.alloc(u8, self.strlen());
         const gecos_start = self.name.len;
         const home_start = gecos_start + self.gecos.len;
-        const shell_start = home_start + self.shell.len;
+        const shell_start = home_start + self.home.len;
         mem.copy(u8, stringdata[0..self.name.len], self.name);
         mem.copy(u8, stringdata[gecos_start..], self.gecos);
         mem.copy(u8, stringdata[home_start..], self.home);