From df8ab1fbbfaeee618272f63a66eca6ffb0cebd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Sun, 27 Feb 2022 15:49:00 +0200 Subject: [PATCH] PackedGroup --- README.md | 2 +- src/{varint.zig => compress.zig} | 14 ++- src/group.zig | 144 +++++++++++++++++++++++++++++-- src/test_main.zig | 3 +- src/user.zig | 86 ++++++++---------- src/validate.zig | 17 ++++ 6 files changed, 199 insertions(+), 67 deletions(-) rename src/{varint.zig => compress.zig} (88%) create mode 100644 src/validate.zig diff --git a/README.md b/README.md index 56f2601..322629b 100644 --- a/README.md +++ b/README.md @@ -358,7 +358,7 @@ STATUS SECTION SIZE DESCRIPTION idx_username2gids len(user)*29/8 bdz->offset Username2gids ✅ ShellIndex len(shells)*2 Shell index array ✅ ShellBlob <= 4032 Shell data blob (max 63*64 bytes) - Groups ? packed Group entries (8b padding) +✅ Groups ? packed Group entries (8b padding) ✅ Users ? packed User entries (8b padding) Groupmembers ? per-group memberlist (32b padding) Username2gids ? Per-user gidlist entries (8b padding) diff --git a/src/varint.zig b/src/compress.zig similarity index 88% rename from src/varint.zig rename to src/compress.zig index 8f0a925..d0149c3 100644 --- a/src/varint.zig +++ b/src/compress.zig @@ -13,7 +13,7 @@ pub const Varint = struct { bytesRead: usize, }; -const MaxVarintLen64 = 10; +const maxVarintLen64 = 10; // https://golang.org/pkg/encoding/binary/#Uvarint pub fn uvarint(buf: []const u8) error{Overflow}!Varint { @@ -21,14 +21,14 @@ pub fn uvarint(buf: []const u8) error{Overflow}!Varint { var s: u6 = 0; for (buf) |b, i| { - if (i == MaxVarintLen64) { - // Catch byte reads past MaxVarintLen64. + if (i == maxVarintLen64) { + // Catch byte reads past maxVarintLen64. // See issue https://golang.org/issues/41185 return error.Overflow; } if (b < 0x80) { - if (i == MaxVarintLen64 - 1 and b > 1) { + if (i == maxVarintLen64 - 1 and b > 1) { return error.Overflow; } return Varint{ .value = x | (@as(u64, b) << s), .bytesRead = i + 1 }; @@ -80,7 +80,7 @@ const tests = [_]u64{ test "uvarint" { for (tests) |x| { - var buf: [MaxVarintLen64]u8 = undefined; + var buf: [maxVarintLen64]u8 = undefined; const n = putUvarint(buf[0..], x); const got = try uvarint(buf[0..n]); @@ -89,10 +89,6 @@ test "uvarint" { } } -const overflowTest = struct { - arr: []const u8, -}; - test "overflow" { for ([_][]const u8{ &[_]u8{ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x2 }, diff --git a/src/group.zig b/src/group.zig index b1e2087..2d88b7b 100644 --- a/src/group.zig +++ b/src/group.zig @@ -1,15 +1,114 @@ const std = @import("std"); -const PackedGroup = packed struct { - gid: u32, - members_offset: u27, - groupname_len: u5, -}; +const pad = @import("padding.zig"); +const validate = @import("validate.zig"); +const InvalidRecord = validate.InvalidRecord; + +const mem = std.mem; +const Allocator = mem.Allocator; +const ArrayList = std.ArrayList; const Group = struct { gid: u32, name: []const u8, - members: std.BufSet, + members_offset: u32, +}; + +const PackedGroup = struct { + const alignmentBits = 3; + + const Inner = packed struct { + gid: u32, + members_offset: u32, + groupname_len: u8, + + pub fn groupnameLen(self: *const Inner) usize { + return self.groupname_len + 1; + } + }; + + inner: *const Inner, + groupdata: []const u8, + + pub const Entry = struct { + group: PackedGroup, + next: ?[]const u8, + }; + + pub fn fromBytes(bytes: []const u8) Entry { + const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]); + const endBlob = @sizeOf(Inner) + inner.groupnameLen(); + const nextStart = pad.roundUp(usize, alignmentBits, endBlob); + + var next: ?[]const u8 = null; + if (nextStart < bytes.len) { + next = bytes[nextStart..]; + } + + return Entry{ + .group = PackedGroup{ + .inner = inner, + .groupdata = bytes[@sizeOf(Inner)..endBlob], + }, + .next = next, + }; + } + + const packErr = validate.InvalidRecord || Allocator.Error; + fn validateUtf8(s: []const u8) InvalidRecord!void { + if (!std.unicode.utf8ValidateSlice(s)) { + return error.InvalidRecord; + } + } + + pub const Iterator = struct { + section: ?[]const u8, + + pub fn next(it: *Iterator) ?PackedGroup { + if (it.section) |section| { + const entry = fromBytes(section); + it.section = entry.next; + return entry.group; + } + return null; + } + }; + + pub fn iterator(section: []const u8) Iterator { + return Iterator{ .section = section }; + } + + pub fn gid(self: *const PackedGroup) u32 { + return self.inner.gid; + } + + pub fn membersOffset(self: *const PackedGroup) u32 { + return self.inner.members_offset; + } + + pub fn name(self: *const PackedGroup) []const u8 { + return self.groupdata; + } + + pub fn packTo( + arr: *ArrayList(u8), + group: Group, + ) packErr!void { + const groupname_len = try validate.downCast(u5, group.name.len - 1); + + try validate.utf8(group.name); + + const inner = Inner{ + .gid = group.gid, + .members_offset = group.members_offset, + .groupname_len = groupname_len, + }; + + try arr.*.appendSlice(mem.asBytes(&inner)); + try arr.*.appendSlice(group.name); + + try pad.arrayList(arr, alignmentBits); + } }; const testing = std.testing; @@ -17,3 +116,36 @@ const testing = std.testing; test "PackedGroup alignment" { try testing.expectEqual(@sizeOf(PackedGroup) * 8, @bitSizeOf(PackedGroup)); } + +test "construct PackedGroups" { + var buf = ArrayList(u8).init(testing.allocator); + defer buf.deinit(); + + const groups = [_]Group{ + Group{ + .gid = 1000, + .name = "sudo", + .members_offset = 1, + }, + Group{ + .gid = std.math.maxInt(u32), + .name = "Name" ** 8, // 32 + .members_offset = std.math.maxInt(u32), + }, + }; + + for (groups) |group| { + try PackedGroup.packTo(&buf, group); + } + + var i: u29 = 0; + { + var it = PackedGroup.iterator(buf.items); + while (it.next()) |group| : (i += 1) { + try testing.expectEqual(groups[i].gid, group.gid()); + try testing.expectEqualStrings(groups[i].name, group.name()); + try testing.expectEqual(groups[i].members_offset, group.membersOffset()); + } + try testing.expectEqual(groups.len, i); + } +} diff --git a/src/test_main.zig b/src/test_main.zig index 444b1dd..b4d3101 100644 --- a/src/test_main.zig +++ b/src/test_main.zig @@ -4,8 +4,9 @@ test "turbonss test suite" { _ = @import("header.zig"); _ = @import("user.zig"); _ = @import("group.zig"); + _ = @import("validate.zig"); _ = @import("padding.zig"); - _ = @import("varint.zig"); + _ = @import("compress.zig"); _ = @import("cmph.zig"); _ = @import("bdz.zig"); } diff --git a/src/user.zig b/src/user.zig index 39a7b1b..b2e6e9a 100644 --- a/src/user.zig +++ b/src/user.zig @@ -1,11 +1,13 @@ const std = @import("std"); + const pad = @import("padding.zig"); +const validate = @import("validate.zig"); +const InvalidRecord = validate.InvalidRecord; const assert = std.debug.assert; const mem = std.mem; const Allocator = mem.Allocator; const ArrayList = std.ArrayList; - // Idx2ShellProto is a function prototype that, given a shell's index (in // global shell section), will return a shell string. Matches ShellReader.get. const Idx2ShellProto = fn (u6) []const u8; @@ -28,20 +30,20 @@ fn packedUser(immutable: bool) type { return struct { const Self = @This(); - const AlignmentBits = 3; + const alignmentBits = 3; const shell2idxProto = fn ([]const u8) ?u6; const InnerSize = @divExact(@bitSizeOf(Inner), 8); const Inner = packed struct { uid: u32, gid: u32, - shell_here: bool, + additional_gids_offset: u29, shell_len_or_idx: u6, + shell_here: bool, home_len: u6, name_is_a_suffix: bool, name_len: u5, gecos_len: u8, - additional_gids_offset: u29, fn homeLen(self: *const Inner) usize { return @as(u32, self.home_len) + 1; @@ -118,7 +120,7 @@ fn packedUser(immutable: bool) type { const startBlob = InnerSize; const endBlob = startBlob + inner.blobLength(); - const nextStart = pad.roundUp(usize, AlignmentBits, endBlob); + const nextStart = pad.roundUp(usize, alignmentBits, endBlob); var next: ?Bytes = null; if (nextStart < bytes.len) { next = bytes[nextStart..]; @@ -133,25 +135,31 @@ fn packedUser(immutable: bool) type { }; } - pub const errInvalid = error{InvalidRecord}; - fn downCast(comptime T: type, n: u64) errInvalid!T { - return std.math.cast(T, n) catch |err| switch (err) { - error.Overflow => { - return error.InvalidRecord; - }, - }; - } + pub const Iterator = struct { + section: ?Bytes, + shellIndex: Idx2ShellProto, - fn validateUtf8(s: []const u8) errInvalid!void { - if (!std.unicode.utf8ValidateSlice(s)) { - return error.InvalidRecord; + pub fn next(it: *Iterator) ?Self { + if (it.section) |section| { + const entry = Self.fromBytes(section); + it.section = entry.next; + return entry.user; + } + return null; } + }; + + pub fn iterator(section: Bytes, idxFn: Idx2ShellProto) Iterator { + return Iterator{ + .section = section, + .shellIndex = idxFn, + }; } // packTo packs the User record and copies it to the given byte slice. The // slice must have at least maxRecordSize() bytes available. // The slice is passed as a pointer, so it can be mutated. - const packErr = errInvalid || Allocator.Error; + const packErr = InvalidRecord || Allocator.Error; pub fn packTo( arr: *ArrayList(u8), user: User, @@ -159,15 +167,15 @@ fn packedUser(immutable: bool) type { ) packErr!void { // function arguments are consts. We need to mutate the underlying // slice, so passing it via pointer instead. - const home_len = try downCast(u6, user.home.len - 1); - const name_len = try downCast(u5, user.name.len - 1); - const shell_len = try downCast(u6, user.shell.len - 1); - const gecos_len = try downCast(u8, user.gecos.len); + const home_len = try validate.downCast(u6, user.home.len - 1); + const name_len = try validate.downCast(u5, user.name.len - 1); + const shell_len = try validate.downCast(u6, user.shell.len - 1); + const gecos_len = try validate.downCast(u8, user.gecos.len); - try validateUtf8(user.home); - try validateUtf8(user.name); - try validateUtf8(user.shell); - try validateUtf8(user.gecos); + try validate.utf8(user.home); + try validate.utf8(user.name); + try validate.utf8(user.shell); + try validate.utf8(user.gecos); const inner = Inner{ .uid = user.uid, @@ -196,7 +204,7 @@ fn packedUser(immutable: bool) type { try arr.*.appendSlice(user.shell); } - try pad.arrayList(arr, AlignmentBits); + try pad.arrayList(arr, alignmentBits); } // maxSize is the maximum number of records a PackedUser can take @@ -208,7 +216,7 @@ fn packedUser(immutable: bool) type { std.math.maxInt(u5) + 1 + // name std.math.maxInt(u6) + 1 + // shell std.math.maxInt(u8); // gecos - return pad.roundUp(u64, AlignmentBits, unpadded); + return pad.roundUp(u64, alignmentBits, unpadded); } } @@ -256,34 +264,13 @@ fn packedUser(immutable: bool) type { } self.inner.additional_gids_offset = new; } - - pub const Iterator = struct { - section: ?Bytes, - shellIndex: Idx2ShellProto, - - pub fn next(it: *Iterator) ?Self { - if (it.section) |section| { - const entry = Self.fromBytes(section); - it.section = entry.next; - return entry.user; - } - return null; - } - }; - - pub fn iterator(section: Bytes, idxFn: Idx2ShellProto) Iterator { - return Iterator{ - .section = section, - .shellIndex = idxFn, - }; - } }; } const testing = std.testing; test "PackedUser internal and external alignment" { - // External padding (AlignmentBits) must be higher or equal to + // External padding (alignmentBits) must be higher or equal to // the "internal" PackedUser alignment. By aligning PackedUser we are also // working around https://github.com/ziglang/zig/issues/10958 ; PackedUser // cannot be converted from/to [@bitSizeOf(PackedUser)/8]u8; @@ -349,7 +336,6 @@ test "construct PackedUser section" { var i: u29 = 0; { var it = PackedUserConst.iterator(buf.items, testShell); - i = 0; while (it.next()) |user| : (i += 1) { try testing.expectEqual(users[i].uid, user.uid()); try testing.expectEqual(users[i].gid, user.gid()); diff --git a/src/validate.zig b/src/validate.zig new file mode 100644 index 0000000..24af3fd --- /dev/null +++ b/src/validate.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +pub const InvalidRecord = error{InvalidRecord}; + +pub fn downCast(comptime T: type, n: u64) InvalidRecord!T { + return std.math.cast(T, n) catch |err| switch (err) { + error.Overflow => { + return error.InvalidRecord; + }, + }; +} + +pub fn utf8(s: []const u8) InvalidRecord!void { + if (!std.unicode.utf8ValidateSlice(s)) { + return error.InvalidRecord; + } +}