From c78a2c0def16e68a4eaabe768a9014299fdec8c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Thu, 10 Mar 2022 18:50:40 +0200 Subject: [PATCH] Implement Groupmembers and Groups section without tests --- README.md | 2 +- src/group.zig | 12 +++++----- src/sections.zig | 60 ++++++++++++++++++++++++++++++++++++++++++++++-- src/user.zig | 46 +++++-------------------------------- 4 files changed, 71 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 5da60e2..98418ed 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ Section creation order: 1. ✅ `shellIndex`, `shellBlob`. No dependencies. 1. ✅ userGids. No dependencies. 1. ✅ Users. Requires `userGids` and shell. -1. Groupmembers. Requires Users. +1. ✅ Groupmembers. Requires Users. 1. Groups. Requires Groupmembers. 1. `idx_*`. Requires offsets to Groups and Users. 1. Header. diff --git a/src/group.zig b/src/group.zig index 1f854e8..1377b33 100644 --- a/src/group.zig +++ b/src/group.zig @@ -31,14 +31,14 @@ pub const Group = struct { } }; -const GroupStored = struct { +pub const GroupStored = struct { gid: u32, name: []const u8, members_offset: u64, }; -const PackedGroup = struct { - const alignment_bits = 3; +pub const PackedGroup = struct { + pub const alignment_bits = 3; const Inner = packed struct { gid: u32, @@ -119,6 +119,7 @@ const PackedGroup = struct { arr: *ArrayList(u8), group: GroupStored, ) packErr!void { + std.debug.assert(arr.items.len & 7 == 0); const groupname_len = try validate.downCast(u5, group.name.len - 1); try validate.utf8(group.name); const inner = Inner{ @@ -129,7 +130,6 @@ const PackedGroup = struct { try arr.*.appendSlice(mem.asBytes(&inner)); try arr.*.appendSlice(group.name); try compress.appendUvarint(arr, group.members_offset); - try pad.arrayList(arr, alignment_bits); } }; @@ -142,9 +142,8 @@ pub fn someMembers( ) Allocator.Error!BufSet { var bufset = BufSet.init(allocator); errdefer bufset.deinit(); - for (members) |member| { + for (members) |member| try bufset.insert(member); - } return bufset; } @@ -171,6 +170,7 @@ test "construct PackedGroups" { for (groups) |group| { try PackedGroup.packTo(&buf, group); + try pad.arrayList(&buf, PackedGroup.alignment_bits); } var i: u29 = 0; diff --git a/src/sections.zig b/src/sections.zig index dd02817..ad46184 100644 --- a/src/sections.zig +++ b/src/sections.zig @@ -243,7 +243,7 @@ pub fn usersSection( ) error{ OutOfMemory, Overflow, InvalidRecord }!UsersSection { var idx2offset = try allocator.alloc(u32, corpus.users.len); errdefer allocator.free(idx2offset); - // as of writing each user takes 15 bytes + strings + padding, padded to + // as of writing each user takes 12 bytes + blobs + padding, padded to // 8 bytes. 24 is an optimistic lower bound for an average record size. var blob = try ArrayList(u8).initCapacity(allocator, 24 * corpus.users.len); errdefer blob.deinit(); @@ -257,6 +257,7 @@ pub fn usersSection( gids.idx2offset[i], shells.indices, ); + try pad.arrayList(&blob, userImport.PackedUserHash.alignment_bits); } return UsersSection{ .idx2offset = idx2offset, @@ -317,6 +318,48 @@ pub fn groupMembers( }; } +pub const GroupsSection = struct { + // group index -> offset in blob + idx2offset: []const u32, + blob: []const u8, + + pub fn deinit(self: *GroupsSection, allocator: Allocator) void { + allocator.free(self.idx2offset); + allocator.free(self.blob); + self.* = undefined; + } +}; + +pub fn groupsSection( + allocator: Allocator, + corpus: *const Corpus, + members_offset: []const u64, +) error{ OutOfMemory, Overflow, InvalidRecord }!GroupsSection { + var idx2offset = try allocator.alloc(u32, corpus.groups.len); + errdefer allocator.free(idx2offset); + + var blob = try ArrayList(u8).initCapacity(allocator, 8 * corpus.groups.len); + errdefer blob.deinit(); + + for (corpus.groups) |group, i| { + const group_offset = try math.cast(u32, blob.items.len); + std.debug.assert(group_offset & 7 == 0); + idx2offset[i] = group_offset; + const group_stored = groupImport.GroupStored{ + .gid = group.gid, + .name = group.name, + .members_offset = members_offset[i], + }; + try groupImport.PackedGroup.packTo(&blob, group_stored); + try pad.arrayList(&blob, groupImport.PackedGroup.alignment_bits); + } + + return GroupsSection{ + .idx2offset = idx2offset, + .blob = blob.toOwnedSlice(), + }; +} + // cmpUser compares two users for sorting. By username's utf8 codepoints, ascending. fn cmpUser(_: void, a: User, b: User) bool { var utf8_a = (unicode.Utf8View.init(a.name) catch unreachable).iterator(); @@ -352,6 +395,7 @@ pub const AllSections = struct { shell_blob: []const u8, user_gids: UserGids, group_members: GroupMembers, + groups: GroupsSection, pub fn init( allocator: Allocator, @@ -376,6 +420,11 @@ pub const AllSections = struct { corpus, users.idx2offset, ); + const groups = try groupsSection( + allocator, + corpus, + group_members.idx2offset, + ); return AllSections{ .allocator = allocator, .bdz_gid = bdz_gid, @@ -388,6 +437,7 @@ pub const AllSections = struct { .user_gids = user_gids, .users = users, .group_members = group_members, + .groups = groups, }; } @@ -400,6 +450,7 @@ pub const AllSections = struct { self.user_gids.deinit(self.allocator); self.users.deinit(self.allocator); self.group_members.deinit(self.allocator); + self.groups.deinit(self.allocator); self.* = undefined; } }; @@ -510,7 +561,7 @@ test "test corpus" { try testing.expectEqual(groupsOfVidmantas[2], g_all); } -test "test group members" { +test "test groups and group members" { const allocator = testing.allocator; var corpus = try testCorpus(allocator); defer corpus.deinit(); @@ -529,7 +580,11 @@ test "test group members" { const want_user_offset = sections.users.idx2offset[user_idx]; try testing.expectEqual(got_user_offset, want_user_offset); } + try testing.expectEqual(it.next(), null); } + + //var it = userImport.PackedUserHash.iterator(sections.groups.blob); + //_ = it; } test "userGids" { @@ -555,6 +610,7 @@ test "userGids" { while (try it.next()) |gid| : (i += 1) { try testing.expectEqual(gid, corpus.groups[groups[i]].gid); } + try testing.expectEqual(i, groups.len); } } diff --git a/src/user.zig b/src/user.zig index f94370e..91a5a1f 100644 --- a/src/user.zig +++ b/src/user.zig @@ -84,7 +84,7 @@ fn packedUser(comptime ShellIndexType: type) type { return struct { const Self = @This(); - const alignmentBits = 3; + pub const alignment_bits = 3; const Inner = packed struct { uid: u32, @@ -165,7 +165,7 @@ fn packedUser(comptime ShellIndexType: type) type { const gids_offset = try compress.uvarint(bytes[end_strings..]); const end_blob = end_strings + gids_offset.bytes_read; - const nextStart = pad.roundUp(usize, alignmentBits, end_blob); + const nextStart = pad.roundUp(usize, alignment_bits, end_blob); var next: ?[]const u8 = null; if (nextStart < bytes.len) next = bytes[nextStart..]; @@ -207,6 +207,7 @@ fn packedUser(comptime ShellIndexType: type) type { additional_gids_offset: u64, idxFn: ShellIndexType, ) error{ InvalidRecord, OutOfMemory }!void { + std.debug.assert(arr.items.len & 7 == 0); // function arguments are consts. We need to mutate the underlying // slice, so passing it via pointer instead. const home_len = try validate.downCast(u6, user.home.len - 1); @@ -231,8 +232,6 @@ fn packedUser(comptime ShellIndexType: type) type { }; const innerBytes = mem.asBytes(&inner); - // innerBytes.len is longer than @sizeOf(Inner). We want to copy - // only the @sizeOf(Inner)-number of bytes. try arr.*.appendSlice(innerBytes[0..@sizeOf(Inner)]); try arr.*.appendSlice(user.home); @@ -242,20 +241,6 @@ fn packedUser(comptime ShellIndexType: type) type { if (inner.shell_here) try arr.*.appendSlice(user.shell); try compress.appendUvarint(arr, additional_gids_offset); - try pad.arrayList(arr, alignmentBits); - } - - // maxSize is the maximum number of records a PackedUser can take - // (struct + strings). - pub fn maxSize() usize { - comptime { - const unpadded = @sizeOf(Inner) + - math.maxInt(u6) + 1 + // home - math.maxInt(u5) + 1 + // name - math.maxInt(u6) + 1 + // shell - math.maxInt(u8); // gecos - return pad.roundUp(u64, alignmentBits, unpadded); - } } pub fn uid(self: Self) u32 { @@ -360,8 +345,10 @@ test "construct PackedUser section" { .home = "/", .shell = "/", } }; - for (users) |user| + for (users) |user| { try PackedUserTest.packTo(&buf, user, math.maxInt(u64), TestShellIndex{}); + try pad.arrayList(&buf, PackedUserHash.alignment_bits); + } var i: u29 = 0; var it1 = PackedUserTest.iterator(buf.items, testShell); @@ -380,27 +367,6 @@ test "construct PackedUser section" { try testing.expectEqual(users.len, i); } -test "PackedUser.maxSize()" { - // TODO(motiejus) try using a slice that points to an array in stack. - // As of writing, I am getting a stack smashing error. - var buf = try ArrayList(u8).initCapacity( - testing.allocator, - PackedUserHash.maxSize(), - ); - defer buf.deinit(); - - const largeUser = User{ - .uid = math.maxInt(u32), - .gid = math.maxInt(u32), - .name = "Name" ** 8, // 32 - .gecos = "Gecos" ** 51, // 255 - .home = "Home" ** 16, // 64 - .shell = "She.LllL" ** 8, // 64 - }; - try PackedUserTest.packTo(&buf, largeUser, math.maxInt(u29), TestShellIndex{}); - try testing.expectEqual(PackedUserTest.maxSize(), buf.items.len); -} - test "User.clone" { var allocator = testing.allocator; const user = User{