From e06cac38b27ab62537f43340c27bf3b1ac92c493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Tue, 22 Feb 2022 15:16:45 +0200 Subject: [PATCH] remove 3 bytes from PackedUser additional_gids will be stored separately. --- README.md | 45 +++++++++++++++++++++++++-------------------- src/user.zig | 25 ++++++------------------- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index f29d11a..f6bb396 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ Now that we've sketched the implementation of `id(3)`, it's clearer to understand which operations need to be fast; in order of importance: 1. lookup gid -> group info (this is on hot path in id) without members. -2. lookup username -> user's group memberships. +2. lookup username -> user's groups. 3. lookup uid -> user. 4. lookup groupname -> group. 5. lookup username -> user. @@ -325,7 +325,7 @@ pseudo-code: ``` const PackedList = struct { length: varint, - members: [length]varint + members: [length]varint, } const Groupmembers = PackedList; const AdditionalGids = PackedList; @@ -333,33 +333,38 @@ const AdditionalGids = PackedList; A packed list is a list of varints. +Section `AdditionalGidsIndex` stores an index from `hash(username)` to `offset` +in AdditionalGids. + Complete file structure ----------------------- `idx_*` sections are of type `[]PackedIntArray(u29)` and are pointing to the respective `Groups` and `Users` entries (from the beginning of the respective -section). Since User and Group records are 8-byte aligned, 3 bits are saved -from every element. +section). Since User and Group records are 8-byte aligned, 3 bits can be saved +from every element. However, since the header easily fits to 64 bytes, we are +storing plain `u32` for easier inspection. Each section is padded to 64 bytes. ``` -SECTION SIZE DESCRIPTION -Header 48 see "Turbonss header" section -bdz_gid2group ? gid->group bdz -bdz_uid2user ? uid->user bdz -bdz_groupname2group ? groupname->group bdz -bdz_name2user ? username->user bdz -idx_gid2group len(group)*4*29/32 bdz->offset gid2group -idx_groupname2group len(group)*4*29/32 bdz->offset groupname2group -idx_uid2user len(user)*4*29/32 bdz->offset uid2user -idx_name2user len(user)*4*29/32 bdz->offset name2user -ShellIndex len(shells)*2 Shell index array -ShellBlob <= 4032 Shell data blob (max 63*64 bytes) -Groups ? packed Group entries (8b padding) -Users ? packed User entries (8b padding) -groupmembers ? per-group memberlist (32b padding) -additional_gids ? per-user grouplist (8b padding) +SECTION SIZE DESCRIPTION +Header 48 see "Turbonss header" section +bdz_gid2group ? gid->group bdz +bdz_uid2user ? uid->user bdz +bdz_groupname2group ? groupname->group bdz +bdz_name2user ? username->user bdz +idx_gid2group len(group)*32 bdz->offset gid2group +idx_groupname2group len(group)*32 bdz->offset groupname2group +idx_uid2user len(user)*32 bdz->offset uid2user +idx_name2user len(user)*32 bdz->offset name2user +idx_username2gids len(user)*32 Per-user gidlist index +ShellIndex len(shells)*2 Shell index array +ShellBlob <= 4032 Shell data blob (max 63*64 bytes) +Groups ? packed Group entries (8b padding) +Users ? packed User entries (8b padding) +Groupmembers ? per-group memberlist (32b padding) +AdditionalGids ? Per-user gidlist entries ``` [git-subtrac]: https://apenwarr.ca/log/20191109 diff --git a/src/user.zig b/src/user.zig index ec455d4..de90a64 100644 --- a/src/user.zig +++ b/src/user.zig @@ -9,13 +9,13 @@ const cast = std.math.cast; pub const PackedUser = packed struct { uid: u32, gid: u32, - additional_gids_offset: u29, shell_here: bool, shell_len_or_idx: u6, home_len: u6, name_is_a_suffix: bool, name_len: u5, gecos_len: u8, + padding: u5, // blobSize returns the length of the blob storing string values. pub fn blobLength(self: *const PackedUser) usize { @@ -93,13 +93,13 @@ pub const UserWriter = struct { var puser = PackedUser{ .uid = user.uid, .gid = user.gid, - .additional_gids_offset = std.math.maxInt(u29), // needs second pass .shell_here = self.shellIndexFn(user.shell) == null, .shell_len_or_idx = self.shellIndexFn(user.shell) orelse shell_len, .home_len = home_len, .name_is_a_suffix = std.mem.endsWith(u8, user.home, user.name), .name_len = name_len, .gecos_len = gecos_len, + .padding = 0, }; try self.appendTo.appendSlice(std.mem.asBytes(&puser)); @@ -144,12 +144,10 @@ pub const UserReader = struct { if (it.index == it.ur.blob.len) return null; assert(it.index < it.ur.blob.len); - // https://github.com/ziglang/zig/issues/1095 - const packedUserSizeHere = @sizeOf(PackedUser); - const endUser = it.index + packedUserSizeHere; + const endUser = it.index + @sizeOf(PackedUser); var packedUser = std.mem.bytesAsValue( PackedUser, - it.ur.blob[it.index..endUser][0..packedUserSizeHere], + it.ur.blob[it.index..endUser][0..@sizeOf(PackedUser)], ); const startBlob = endUser; const endBlob = startBlob + packedUser.blobLength(); @@ -217,19 +215,8 @@ pub const UserReader = struct { const testing = std.testing; -test "PackedUser alignment" { - // byte-aligned - try testing.expectEqual(0, @rem(@bitSizeOf(PackedUser), 8)); - - const bytes = @divExact(@bitSizeOf(PackedUser), 8); - - // External padding (PackedUserAlignmentBits) must be higher or equal to - // the "internal" PackedUser alignment. By aligning PackedUser we are also - // working around https://github.com/ziglang/zig/issues/10958 ; PackedUser - // cannot be converted from/to [@bitSizeOf(PackedUser)/8]u8; - // asBytes/bytesAsValue use @sizeOf, which is larger. Now we are putting no - // more than 1, but it probably could be higher. - try testing.expect(@sizeOf(PackedUser) - bytes <= 1); +test "PackedUser internal and external alignment" { + try testing.expectEqual(@bitSizeOf(PackedUser), @sizeOf(PackedUser) * 8); } fn testShellIndex(shell: []const u8) ?u6 {