diff --git a/README.md b/README.md index d6ec01b..2014a16 100644 --- a/README.md +++ b/README.md @@ -372,11 +372,10 @@ Section creation order: 1. ✅ `bdz_*`. No depdendencies. 1. ✅ `shellIndex`, `shellBlob`. No dependencies. -1. UserGids. No dependencies. -1. Users, but without `additional_gids_offset`. No dependencies. -1. Groupmembers. Depends on Users, ex. `additional_gids_offset`. +1. ✅ userGids. No dependencies. +1. Users. Requires `userGids`. +1. Groupmembers. Requires Users. 1. Groups. Requires Groupmembers. -1. Mutate `Users.additional_gids_offset`. Requires Groupmembers and ShellIndex. 1. `idx_*`. Requires offsets to Groups and Users. 1. Header. diff --git a/src/compress.zig b/src/compress.zig index 938d859..4851d26 100644 --- a/src/compress.zig +++ b/src/compress.zig @@ -5,6 +5,9 @@ // golang's varint implementation. const std = @import("std"); +const ArrayList = std.ArrayList; +const Allocator = std.mem.Allocator; + // compresses a strictly incrementing sorted slice of integers using delta // compression. Compression is in-place. pub fn deltaCompress(comptime T: type, elems: []T) error{NotSorted}!void { @@ -48,7 +51,7 @@ test "delta compress/decompress" { .{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } }, }; for (tests) |t| { - var arr = try std.ArrayList(u8).initCapacity( + var arr = try ArrayList(u8).initCapacity( testing.allocator, t.input.len, ); @@ -69,7 +72,7 @@ test "delta compression negative tests" { &[_]u8{ 0, 1, 1 }, &[_]u8{ 0, 1, 2, 1 }, }) |t| { - var arr = try std.ArrayList(u8).initCapacity(testing.allocator, t.len); + var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len); defer arr.deinit(); try arr.appendSlice(t); try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items)); @@ -81,7 +84,7 @@ test "delta decompress overflow" { &[_]u8{ 255, 0 }, &[_]u8{ 0, 128, 127 }, }) |t| { - var arr = try std.ArrayList(u8).initCapacity(testing.allocator, t.len); + var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len); defer arr.deinit(); try arr.appendSlice(t); try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items)); @@ -142,24 +145,31 @@ pub fn putUvarint(buf: []u8, x: u64) usize { return i + 1; } -test "uvarint" { - const uvarint_tests = [_]u64{ - 0, - 1, - 2, - 10, - 20, - 63, - 64, - 65, - 127, - 128, - 129, - 255, - 256, - 257, - 1 << 63 - 1, - }; +pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void { + var buf: [maxVarintLen64]u8 = undefined; + const n = putUvarint(&buf, x); + try arr.appendSlice(buf[0..n]); +} + +const uvarint_tests = [_]u64{ + 0, + 1, + 2, + 10, + 20, + 63, + 64, + 65, + 127, + 128, + 129, + 255, + 256, + 257, + 1 << 63 - 1, +}; + +test "putUvarint/uvarint" { for (uvarint_tests) |x| { var buf: [maxVarintLen64]u8 = undefined; const n = putUvarint(buf[0..], x); @@ -170,6 +180,18 @@ test "uvarint" { } } +test "appendUvarint" { + for (uvarint_tests) |x| { + var buf = ArrayList(u8).init(testing.allocator); + defer buf.deinit(); + + try appendUvarint(&buf, x); + const got = try uvarint(buf.items); + + try testing.expectEqual(x, got.value); + } +} + test "overflow" { for ([_][]const u8{ &[_]u8{ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2 }, diff --git a/src/sections.zig b/src/sections.zig index 9c4c012..6fa1e04 100644 --- a/src/sections.zig +++ b/src/sections.zig @@ -108,6 +108,7 @@ const Corpus = struct { var username2groups = StringHashMap( ArrayListUnmanaged(*const Group), ).init(baseAllocator); + for (groups) |*group| { var members = try allocator.alloc(*const User, group.members.count()); members.len = 0; @@ -143,9 +144,11 @@ const Corpus = struct { sort.sort(*const Group, userGroups.items, {}, cmpGroupPtr); var username2groups_final = StringHashMap([]*const Group).init(allocator); - for (users) |user| { - const userGroups = username2groups.get(user.name).?.toOwnedSlice(allocator); - try username2groups_final.put(user.name, userGroups); + var it = username2groups.iterator(); + while (it.next()) |elem| { + const username = elem.key_ptr.*; + const usergroups = elem.value_ptr.*.toOwnedSlice(allocator); + try username2groups_final.put(username, usergroups); } username2groups.deinit(); @@ -228,6 +231,53 @@ pub const Sections = struct { }; } + pub const UserGids = struct { + // username -> offset in blob + name2offset: StringHashMap(u32), + // compressed user gids blob. A blob contains N <= users.len items, + // an item is: + // len: varint + // gid: [varint]varint, + // ... and the gid list is delta-compressed. + blob: []u8, + }; + + const userGidsErr = Allocator.Error || error{Overflow}; + pub fn userGids(self: *const Sections) userGidsErr!UserGids { + var arena = std.heap.ArenaAllocator.init(self.allocator); + defer arena.deinit(); + var blob = ArrayList(u8).init(self.allocator); + + var name2offset = StringHashMap(u32).init(self.allocator); + for (self.corpus.users) |user| { + const usergroups_maybe = self.corpus.username2groups.get(user.name); + if (usergroups_maybe == null) + continue; + const usergroups = usergroups_maybe.?; + + try name2offset.putNoClobber(user.name, try math.cast(u32, blob.items.len)); + var userBlob = try ArrayList(u8).initCapacity(arena.allocator(), usergroups.len * 2); + var deltaCompressedGids = try arena.allocator().alloc(u32, usergroups.len); + deltaCompressedGids.len = usergroups.len; + for (usergroups) |group, i| { + deltaCompressedGids[i] = group.gid; + } + compress.deltaCompress(u32, deltaCompressedGids) catch |err| switch (err) { + error.NotSorted => unreachable, + }; + try compress.appendUvarint(&userBlob, usergroups.len); + for (deltaCompressedGids) |gid| { + try compress.appendUvarint(&userBlob, gid); + } + try blob.appendSlice(userBlob.toOwnedSlice()); + } + + return UserGids{ + .name2offset = name2offset, + .blob = blob.toOwnedSlice(), + }; + } + pub fn groupMembers(self: *const Sections) Allocator.Error!GroupMembers { var buf: [compress.maxVarintLen64]u8 = undefined; var offsets = ArrayListUnmanaged(usize).initCapacity( @@ -306,6 +356,13 @@ fn testCorpus(allocator: Allocator) !Corpus { .gecos = "", .home = "/", .shell = "/", + }, User{ + .uid = 65534, + .gid = 65534, + .name = "nobody", + .gecos = "nobody", + .home = "/nonexistent", + .shell = "/usr/sbin/nologin", } }; var members1 = try groupImport.someMembers( @@ -348,8 +405,9 @@ test "test corpus" { defer corpus.deinit(); try testing.expectEqualStrings(corpus.users[0].name, "Name" ** 8); - try testing.expectEqualStrings(corpus.users[1].name, "svc-bar"); - try testing.expectEqualStrings(corpus.users[2].name, "vidmantas"); + try testing.expectEqualStrings(corpus.users[1].name, "nobody"); + try testing.expectEqualStrings(corpus.users[2].name, "svc-bar"); + try testing.expectEqualStrings(corpus.users[3].name, "vidmantas"); try testing.expectEqual(corpus.name2user.get("404"), null); try testing.expectEqual(corpus.name2user.get("vidmantas").?.uid, 128); @@ -370,7 +428,8 @@ test "test corpus" { try testing.expectEqual(groupsOfVidmantas[0].gid, 0); try testing.expectEqual(groupsOfVidmantas[1].gid, 128); try testing.expectEqual(groupsOfVidmantas[2].gid, 9999); - try testing.expectEqual(corpus.username2groups.get("404"), null); + try testing.expectEqual(corpus.username2groups.get("nobody"), null); + try testing.expectEqual(corpus.username2groups.get("doesnotexist"), null); } test "test sections" { @@ -392,9 +451,13 @@ test "test sections" { const bdz_username = try sections.bdzUsername(); defer allocator.free(bdz_username); - const shellSections = try sections.shellSections(); - defer allocator.free(shellSections.index); - defer allocator.free(shellSections.blob); + const shell_sections = try sections.shellSections(); + defer allocator.free(shell_sections.index); + defer allocator.free(shell_sections.blob); + + var user_gids = try sections.userGids(); + defer user_gids.name2offset.deinit(); + defer allocator.free(user_gids.blob); } test "pack gids" { diff --git a/src/user.zig b/src/user.zig index 83ab07e..e8d6ee0 100644 --- a/src/user.zig +++ b/src/user.zig @@ -31,7 +31,7 @@ pub const User = struct { const stringdata = try allocator.alloc(u8, self.strlen()); const gecos_start = self.name.len; const home_start = gecos_start + self.gecos.len; - const shell_start = home_start + self.shell.len; + const shell_start = home_start + self.home.len; mem.copy(u8, stringdata[0..self.name.len], self.name); mem.copy(u8, stringdata[gecos_start..], self.gecos); mem.copy(u8, stringdata[home_start..], self.home);