motiejus/turbonss
1
Fork 0
Code Releases Activity

remove 3 bytes from PackedUser

Browse Source 
additional_gids will be stored separately.
This commit is contained in:
Motiejus Jakštys 2022-02-22 15:16:45 +02:00 committed by Motiejus Jakštys
parent 78dc63669d
commit e06cac38b2
2 changed files with 31 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
README.md
View File

@ -130,7 +130,7 @@ Now that we've sketched the implementation of `id(3)`, it's clearer to
understand which operations need to be fast; in order of importance: understand which operations need to be fast; in order of importance:
1. lookup gid -> group info (this is on hot path in id) without members. 1. lookup gid -> group info (this is on hot path in id) without members.
2. lookup username -> user's group memberships. 2. lookup username -> user's groups.
3. lookup uid -> user. 3. lookup uid -> user.
4. lookup groupname -> group. 4. lookup groupname -> group.
5. lookup username -> user. 5. lookup username -> user.
@ -325,7 +325,7 @@ pseudo-code:
``` ```
const PackedList = struct { const PackedList = struct {
length: varint, length: varint,
members: [length]varint members: [length]varint,
} }
const Groupmembers = PackedList; const Groupmembers = PackedList;
const AdditionalGids = PackedList; const AdditionalGids = PackedList;
@ -333,33 +333,38 @@ const AdditionalGids = PackedList;
A packed list is a list of varints. A packed list is a list of varints.
Section `AdditionalGidsIndex` stores an index from `hash(username)` to `offset`
in AdditionalGids.
Complete file structure Complete file structure
----------------------- -----------------------
`idx_*` sections are of type `[]PackedIntArray(u29)` and are pointing to the `idx_*` sections are of type `[]PackedIntArray(u29)` and are pointing to the
respective `Groups` and `Users` entries (from the beginning of the respective respective `Groups` and `Users` entries (from the beginning of the respective
section). Since User and Group records are 8-byte aligned, 3 bits are saved section). Since User and Group records are 8-byte aligned, 3 bits can be saved
from every element. from every element. However, since the header easily fits to 64 bytes, we are
storing plain `u32` for easier inspection.
Each section is padded to 64 bytes. Each section is padded to 64 bytes.
``` ```
SECTION SIZE DESCRIPTION SECTION SIZE DESCRIPTION
Header 48 see "Turbonss header" section Header 48 see "Turbonss header" section
bdz_gid2group ? gid->group bdz bdz_gid2group ? gid->group bdz
bdz_uid2user ? uid->user bdz bdz_uid2user ? uid->user bdz
bdz_groupname2group ? groupname->group bdz bdz_groupname2group ? groupname->group bdz
bdz_name2user ? username->user bdz bdz_name2user ? username->user bdz
idx_gid2group len(group)*4*29/32 bdz->offset gid2group idx_gid2group len(group)*32 bdz->offset gid2group
idx_groupname2group len(group)*4*29/32 bdz->offset groupname2group idx_groupname2group len(group)*32 bdz->offset groupname2group
idx_uid2user len(user)*4*29/32 bdz->offset uid2user idx_uid2user len(user)*32 bdz->offset uid2user
idx_name2user len(user)*4*29/32 bdz->offset name2user idx_name2user len(user)*32 bdz->offset name2user
ShellIndex len(shells)*2 Shell index array idx_username2gids len(user)*32 Per-user gidlist index
ShellBlob <= 4032 Shell data blob (max 63*64 bytes) ShellIndex len(shells)*2 Shell index array
Groups ? packed Group entries (8b padding) ShellBlob <= 4032 Shell data blob (max 63*64 bytes)
Users ? packed User entries (8b padding) Groups ? packed Group entries (8b padding)
groupmembers ? per-group memberlist (32b padding) Users ? packed User entries (8b padding)
additional_gids ? per-user grouplist (8b padding) Groupmembers ? per-group memberlist (32b padding)
AdditionalGids ? Per-user gidlist entries
``` ```
[git-subtrac]: https://apenwarr.ca/log/20191109 [git-subtrac]: https://apenwarr.ca/log/20191109

25
src/user.zig
View File

@ -9,13 +9,13 @@ const cast = std.math.cast;
pub const PackedUser = packed struct { pub const PackedUser = packed struct {
uid: u32, uid: u32,
gid: u32, gid: u32,
additional_gids_offset: u29,
shell_here: bool, shell_here: bool,
shell_len_or_idx: u6, shell_len_or_idx: u6,
home_len: u6, home_len: u6,
name_is_a_suffix: bool, name_is_a_suffix: bool,
name_len: u5, name_len: u5,
gecos_len: u8, gecos_len: u8,
padding: u5,
// blobSize returns the length of the blob storing string values. // blobSize returns the length of the blob storing string values.
pub fn blobLength(self: *const PackedUser) usize { pub fn blobLength(self: *const PackedUser) usize {
@ -93,13 +93,13 @@ pub const UserWriter = struct {
var puser = PackedUser{ var puser = PackedUser{
.uid = user.uid, .uid = user.uid,
.gid = user.gid, .gid = user.gid,
.additional_gids_offset = std.math.maxInt(u29), // needs second pass
.shell_here = self.shellIndexFn(user.shell) == null, .shell_here = self.shellIndexFn(user.shell) == null,
.shell_len_or_idx = self.shellIndexFn(user.shell) orelse shell_len, .shell_len_or_idx = self.shellIndexFn(user.shell) orelse shell_len,
.home_len = home_len, .home_len = home_len,
.name_is_a_suffix = std.mem.endsWith(u8, user.home, user.name), .name_is_a_suffix = std.mem.endsWith(u8, user.home, user.name),
.name_len = name_len, .name_len = name_len,
.gecos_len = gecos_len, .gecos_len = gecos_len,
.padding = 0,
}; };
try self.appendTo.appendSlice(std.mem.asBytes(&puser)); try self.appendTo.appendSlice(std.mem.asBytes(&puser));
@ -144,12 +144,10 @@ pub const UserReader = struct {
if (it.index == it.ur.blob.len) return null; if (it.index == it.ur.blob.len) return null;
assert(it.index < it.ur.blob.len); assert(it.index < it.ur.blob.len);
// https://github.com/ziglang/zig/issues/1095 const endUser = it.index + @sizeOf(PackedUser);
const packedUserSizeHere = @sizeOf(PackedUser);
const endUser = it.index + packedUserSizeHere;
var packedUser = std.mem.bytesAsValue( var packedUser = std.mem.bytesAsValue(
PackedUser, PackedUser,
it.ur.blob[it.index..endUser][0..packedUserSizeHere], it.ur.blob[it.index..endUser][0..@sizeOf(PackedUser)],
); );
const startBlob = endUser; const startBlob = endUser;
const endBlob = startBlob + packedUser.blobLength(); const endBlob = startBlob + packedUser.blobLength();
@ -217,19 +215,8 @@ pub const UserReader = struct {
const testing = std.testing; const testing = std.testing;
test "PackedUser alignment" { test "PackedUser internal and external alignment" {
// byte-aligned try testing.expectEqual(@bitSizeOf(PackedUser), @sizeOf(PackedUser) * 8);
try testing.expectEqual(0, @rem(@bitSizeOf(PackedUser), 8));
const bytes = @divExact(@bitSizeOf(PackedUser), 8);
// External padding (PackedUserAlignmentBits) must be higher or equal to
// the "internal" PackedUser alignment. By aligning PackedUser we are also
// working around https://github.com/ziglang/zig/issues/10958 ; PackedUser
// cannot be converted from/to [@bitSizeOf(PackedUser)/8]u8;
// asBytes/bytesAsValue use @sizeOf, which is larger. Now we are putting no
// more than 1, but it probably could be higher.
try testing.expect(@sizeOf(PackedUser) - bytes <= 1);
} }
fn testShellIndex(shell: []const u8) ?u6 { fn testShellIndex(shell: []const u8) ?u6 {