const std = @import("std"); const os = std.os; const mem = std.mem; const math = std.math; const meta = std.meta; const sort = std.sort; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const ArrayList = std.ArrayList; const AutoHashMap = std.AutoHashMap; const BoundedArray = std.BoundedArray; const ErrCtx = @import("ErrCtx.zig"); const Corpus = @import("Corpus.zig"); const pad = @import("padding.zig"); const compress = @import("compress.zig"); const Group = @import("Group.zig"); const CGroup = Group.CGroup; const PackedGroup = @import("PackedGroup.zig"); const GroupStored = PackedGroup.GroupStored; const User = @import("User.zig"); const CUser = User.CUser; const PackedUser = @import("PackedUser.zig"); const ShellSections = @import("shell.zig").ShellWriter.ShellSections; const ShellReader = @import("shell.zig").ShellReader; const ShellWriter = @import("shell.zig").ShellWriter; const InvalidHeader = @import("header.zig").Invalid; const Header = @import("header.zig").Header; const max_shells = @import("shell.zig").max_shells; const section_length_bits = @import("header.zig").section_length_bits; const section_length = @import("header.zig").section_length; const cmph = @import("cmph.zig"); const bdz = @import("bdz.zig"); const zeroes = &[_]u8{0} ** section_length; const DB = @This(); // All sections, as they end up in the DB. Order is important. header: *const Header, bdz_gid: []const u8, bdz_groupname: []const u8, bdz_uid: []const u8, bdz_username: []const u8, idx_gid2group: []const u32, idx_groupname2group: []const u32, idx_uid2user: []const u32, idx_name2user: []const u32, shell_index: []const u16, shell_blob: []const u8, groups: []const u8, users: []const u8, groupmembers: []const u8, additional_gids: []const u8, pub fn fromCorpus( allocator: Allocator, corpus: *const Corpus, err: *ErrCtx, ) error{ OutOfMemory, InvalidRecord, TooMany }!DB { _ = err; const gids = corpus.groups.items(.gid); const gnames = corpus.groups.items(.name); const uids = corpus.users.items(.uid); const unames = corpus.users.items(.name); const bdz_gid = try cmph.packU32(allocator, gids); errdefer allocator.free(bdz_gid); const bdz_groupname = try cmph.packStr(allocator, gnames); errdefer allocator.free(bdz_groupname); const bdz_uid = try cmph.packU32(allocator, uids); errdefer allocator.free(bdz_uid); const bdz_username = try cmph.packStr(allocator, unames); errdefer allocator.free(bdz_username); var shell = try shellSections(allocator, corpus); defer shell.deinit(); const shell_index = try allocator.dupe(u16, shell.index.constSlice()); errdefer allocator.free(shell_index); const shell_blob = try allocator.dupe(u8, shell.blob.constSlice()); errdefer allocator.free(shell_blob); const additional_gids = try additionalGids(allocator, corpus); errdefer allocator.free(additional_gids.blob); defer allocator.free(additional_gids.idx2offset); const users = try usersSection(allocator, corpus, &additional_gids, &shell); errdefer allocator.free(users.blob); defer allocator.free(users.idx2offset); const groupmembers = try groupMembers(allocator, corpus, users.idx2offset); errdefer allocator.free(groupmembers.blob); defer allocator.free(groupmembers.idx2offset); const groups = try groupsSection(allocator, corpus, groupmembers.idx2offset); errdefer allocator.free(groups.blob); defer allocator.free(groups.idx2offset); const idx_gid2group = try bdzIdx(u32, allocator, bdz_gid, gids, groups.idx2offset); errdefer allocator.free(idx_gid2group); const idx_groupname2group = try bdzIdx([]const u8, allocator, bdz_groupname, gnames, groups.idx2offset); errdefer allocator.free(idx_groupname2group); const idx_uid2user = try bdzIdx(u32, allocator, bdz_uid, uids, users.idx2offset); errdefer allocator.free(idx_uid2user); const idx_name2user = try bdzIdx([]const u8, allocator, bdz_username, unames, users.idx2offset); errdefer allocator.free(idx_name2user); const header = try allocator.create(Header); errdefer allocator.destroy(header); header.* = Header{ .nblocks_shell_blob = nblocks(u8, shell.blob.constSlice()), .num_shells = shell.len, .num_groups = groups.len, .num_users = users.len, .nblocks_bdz_gid = nblocks(u32, bdz_gid), .nblocks_bdz_groupname = nblocks(u32, bdz_groupname), .nblocks_bdz_uid = nblocks(u32, bdz_uid), .nblocks_bdz_username = nblocks(u32, bdz_username), .nblocks_groups = nblocks(u64, groups.blob), .nblocks_users = nblocks(u64, users.blob), .nblocks_groupmembers = nblocks(u64, groupmembers.blob), .nblocks_additional_gids = nblocks(u64, additional_gids.blob), .getgr_bufsize = corpus.getgr_bufsize, .getpw_bufsize = corpus.getpw_bufsize, }; return DB{ .header = header, .bdz_gid = bdz_gid, .bdz_groupname = bdz_groupname, .bdz_uid = bdz_uid, .bdz_username = bdz_username, .idx_gid2group = idx_gid2group, .idx_groupname2group = idx_groupname2group, .idx_uid2user = idx_uid2user, .idx_name2user = idx_name2user, .shell_index = shell_index, .shell_blob = shell_blob, .groups = groups.blob, .users = users.blob, .groupmembers = groupmembers.blob, .additional_gids = additional_gids.blob, }; } pub fn getgrBufsize(self: *const DB) usize { return self.header.getgr_bufsize; } pub fn getpwBufsize(self: *const DB) usize { return self.header.getpw_bufsize; } pub fn deinit(self: *DB, allocator: Allocator) void { allocator.destroy(self.header); allocator.free(self.bdz_gid); allocator.free(self.bdz_groupname); allocator.free(self.bdz_uid); allocator.free(self.bdz_username); allocator.free(self.idx_gid2group); allocator.free(self.idx_groupname2group); allocator.free(self.idx_uid2user); allocator.free(self.idx_name2user); allocator.free(self.shell_index); allocator.free(self.shell_blob); allocator.free(self.groups); allocator.free(self.users); allocator.free(self.groupmembers); allocator.free(self.additional_gids); self.* = undefined; } const DB_fields = meta.fields(DB); pub fn iov(self: *const DB) BoundedArray(os.iovec_const, DB_fields.len * 2) { var result = BoundedArray(os.iovec_const, DB_fields.len * 2).init(0) catch unreachable; inline for (DB_fields) |field| { comptime assertDefinedLayout(field.field_type); const value = @field(self, field.name); const bytes: []const u8 = switch (@TypeOf(value)) { *const Header => mem.asBytes(value), else => mem.sliceAsBytes(value), }; result.appendAssumeCapacity(os.iovec_const{ .iov_base = bytes.ptr, .iov_len = bytes.len, }); const padding = pad.until(usize, section_length_bits, bytes.len); if (padding != 0) result.appendAssumeCapacity(.{ .iov_base = zeroes, .iov_len = padding, }); } return result; } pub const DBNumbers = struct { header: u64, bdz_gid: u64, bdz_groupname: u64, bdz_uid: u64, bdz_username: u64, idx_gid2group: u64, idx_groupname2group: u64, idx_uid2user: u64, idx_name2user: u64, shell_index: u64, shell_blob: u64, groups: u64, users: u64, groupmembers: u64, additional_gids: u64, }; // in blocks pub fn fieldLengths(header: *const Header) DBNumbers { return DBNumbers{ .header = comptime DB.nblocks_n(u64, @sizeOf(Header)), .bdz_gid = header.nblocks_bdz_gid, .bdz_groupname = header.nblocks_bdz_groupname, .bdz_uid = header.nblocks_bdz_uid, .bdz_username = header.nblocks_bdz_username, .idx_gid2group = nblocks_n(u32, header.num_groups * 4), .idx_groupname2group = nblocks_n(u32, header.num_groups * 4), .idx_uid2user = nblocks_n(u32, header.num_users * 4), .idx_name2user = nblocks_n(u32, header.num_users * 4), .shell_index = nblocks_n(u16, header.num_shells * 2), .shell_blob = header.nblocks_shell_blob, .groups = header.nblocks_groups, .users = header.nblocks_users, .groupmembers = header.nblocks_groupmembers, .additional_gids = header.nblocks_additional_gids, }; } // in blocks pub fn fieldOffsets(lengths: DBNumbers) DBNumbers { var result: DBNumbers = undefined; result.header = 0; var offset = comptime nblocks_n(u64, @sizeOf(Header)); inline for (DB_fields[0..]) |field, i| { comptime { assert(mem.eql(u8, field.name, meta.fields(DBNumbers)[i].name)); if (mem.eql(u8, field.name, "header")) continue; } @field(result, field.name) = offset; offset += @field(lengths, field.name); } return result; } pub fn fromBytes(buf: []align(8) const u8) InvalidHeader!DB { const header = try Header.fromBytes(buf[0..@sizeOf(Header)]); const lengths = fieldLengths(header); const offsets = fieldOffsets(lengths); var result: DB = undefined; result.header = header; inline for (DB_fields[1..]) |field| { const start_block = @field(offsets, field.name); const end = (start_block + @field(lengths, field.name)) << section_length_bits; const start = start_block << section_length_bits; const slice_type = meta.Child(field.field_type); const value = mem.bytesAsSlice(slice_type, buf[start..end]); @field(result, field.name) = value; } return result; } pub fn packCGroupNoMembers(group: *const PackedGroup, buf: []u8) error{BufferTooSmall}!CGroup { // first word in buf will be a pointer to null. that probably can be // simplified by writing (0)**word_size to the buffer, but let's pretend // type safety for a moment. const name_start = @sizeOf(?[*:0]const u8); if (name_start > buf.len) return error.BufferTooSmall; var member_ptrs = mem.bytesAsSlice(?[*:0]const u8, buf[0..name_start]); member_ptrs[0] = null; // write name const name = group.name(); if (name_start + name.len + 1 > buf.len) return error.BufferTooSmall; mem.copy(u8, buf[name_start..], name); buf[name_start + name.len] = 0; return CGroup{ .gr_name = buf[name_start .. name_start + name.len :0].ptr, .gr_gid = group.gid(), .gr_mem = member_ptrs.ptr, }; } // The pointer to `arr` gets changed on second iteration, and I don't know why. // Leaving this for better times. //const GroupMembersIter = struct { // vit: compress.VarintSliceIterator, // it: compress.DeltaDecompressionIterator, // total: usize, // arr: []const u8, // // pub fn nextMust(self: *GroupMembersIter) ?u64 { // return self.it.nextMust(); // } //}; // //pub fn groupMembersIter(members_slice: []const u8) GroupMembersIter { // std.debug.print("members_slice.ptr={*}\n", .{members_slice.ptr}); // var vit = compress.varintSliceIteratorMust(members_slice); // var it = compress.deltaDecompressionIterator(&vit); // return GroupMembersIter{ // .arr = members_slice, // .vit = vit, // .it = it, // .total = vit.remaining, // }; //} // dumps PackedGroup to []u8 and returns a CGroup. pub fn packCGroup(self: *const DB, group: *const PackedGroup, buf: []u8) error{BufferTooSmall}!CGroup { const members_slice = self.groupmembers[group.members_offset..]; var vit = compress.varintSliceIteratorMust(members_slice); const num_members = vit.remaining; const ptr_end = @sizeOf(?[*:0]const u8) * (num_members + 1); if (ptr_end > buf.len) return error.BufferTooSmall; var member_ptrs = mem.bytesAsSlice(?[*:0]const u8, buf[0..ptr_end]); member_ptrs[member_ptrs.len - 1] = null; var buf_offset: usize = ptr_end; var it = compress.deltaDecompressionIterator(&vit); var i: usize = 0; while (it.nextMust()) |member_offset| : (i += 1) { const entry = PackedUser.fromBytes(self.users[member_offset << 3 ..]); const start = buf_offset; const name = entry.user.name(); if (buf_offset + name.len + 1 > buf.len) return error.BufferTooSmall; mem.copy(u8, buf[buf_offset..], name); buf_offset += name.len; buf[buf_offset] = 0; buf_offset += 1; // TODO: arr[i] = buf[...] triggers a bug in zig pre-0.10 const terminated = buf[start .. buf_offset - 1 :0]; member_ptrs[i] = terminated; } const name = group.name(); if (buf_offset + name.len + 1 > buf.len) return error.BufferTooSmall; mem.copy(u8, buf[buf_offset..], name); buf[buf_offset + name.len] = 0; return CGroup{ .gr_name = buf[buf_offset .. buf_offset + name.len :0].ptr, .gr_gid = group.gid(), .gr_mem = member_ptrs.ptr, }; } pub fn getGroupByName(self: *const DB, name: []const u8) ?PackedGroup { const idx = bdz.search(self.bdz_groupname, name); if (idx >= self.header.num_groups) return null; const offset = self.idx_groupname2group[idx]; const nbits = PackedGroup.alignment_bits; const group = PackedGroup.fromBytes(self.groups[offset << nbits ..]).group; if (!mem.eql(u8, name, group.name())) return null; return group; } pub fn getGroupByGid(self: *const DB, gid: u32) ?PackedGroup { const idx = bdz.search_u32(self.bdz_gid, gid); if (idx >= self.header.num_groups) return null; const offset = self.idx_gid2group[idx]; const nbits = PackedGroup.alignment_bits; const group = PackedGroup.fromBytes(self.groups[offset << nbits ..]).group; if (gid != group.gid()) return null; return group; } // get a CGroup entry by name. pub fn getgrnam( self: *const DB, name: []const u8, buf: []u8, omit_members: bool, ) error{BufferTooSmall}!?CGroup { const group = self.getGroupByName(name) orelse return null; if (omit_members) return try packCGroupNoMembers(&group, buf) else return try self.packCGroup(&group, buf); } // get a CGroup entry by it's gid. pub fn getgrgid( self: *const DB, gid: u32, buf: []u8, omit_members: bool, ) error{BufferTooSmall}!?CGroup { const group = self.getGroupByGid(gid) orelse return null; if (omit_members) return try packCGroupNoMembers(&group, buf) else return try self.packCGroup(&group, buf); } fn pushStr(str: []const u8, buf: []u8, offset: *usize) [*:0]const u8 { const start = offset.*; mem.copy(u8, buf[start..], str); buf[start + str.len] = 0; offset.* += str.len + 1; return buf[start .. start + str.len :0].ptr; } pub fn shellReader(self: *const DB) ShellReader { return ShellReader{ .index = self.shell_index, .blob = self.shell_blob, }; } pub fn writeUser(self: *const DB, user: PackedUser, buf: []u8) error{BufferTooSmall}!CUser { const name = user.name(); const gecos = user.gecos(); const home = user.home(); const shell = user.shell(self.shellReader()); const strlen = name.len + 1 + gecos.len + 1 + home.len + 1 + shell.len + 1; if (strlen > buf.len) return error.BufferTooSmall; var offset: usize = 0; const pw_name = pushStr(name, buf, &offset); const pw_gecos = pushStr(gecos, buf, &offset); const pw_dir = pushStr(home, buf, &offset); const pw_shell = pushStr(shell, buf, &offset); return CUser{ .pw_name = pw_name, .pw_uid = user.uid(), .pw_gid = user.gid(), .pw_gecos = pw_gecos, .pw_dir = pw_dir, .pw_shell = pw_shell, }; } pub fn getUserByName(self: *const DB, name: []const u8) ?PackedUser { const idx = bdz.search(self.bdz_username, name); // bdz may return a hash that's bigger than the number of users if (idx >= self.header.num_users) return null; const offset = self.idx_name2user[idx]; const nbits = PackedUser.alignment_bits; const user = PackedUser.fromBytes(self.users[offset << nbits ..]).user; if (!mem.eql(u8, name, user.name())) return null; return user; } // get a CUser entry by name. pub fn getpwnam(self: *const DB, name: []const u8, buf: []u8) error{BufferTooSmall}!?CUser { const user = self.getUserByName(name) orelse return null; return try self.writeUser(user, buf); } pub fn getUserByUid(self: *const DB, uid: u32) ?PackedUser { const idx = bdz.search_u32(self.bdz_uid, uid); if (idx >= self.header.num_users) return null; const offset = self.idx_uid2user[idx]; const nbits = PackedUser.alignment_bits; const user = PackedUser.fromBytes(self.users[offset << nbits ..]).user; if (uid != user.uid()) return null; return user; } // get a CUser entry by uid. pub fn getpwuid(self: *const DB, uid: u32, buf: []u8) error{BufferTooSmall}!?CUser { const user = self.getUserByUid(uid) orelse return null; return try self.writeUser(user, buf); } fn shellSections( allocator: Allocator, corpus: *const Corpus, ) error{OutOfMemory}!ShellSections { var popcon = ShellWriter.init(allocator); errdefer popcon.deinit(); for (corpus.users.items(.shell)) |shell| try popcon.put(shell); return popcon.toOwnedSections(max_shells); } const AdditionalGids = struct { // user index -> offset in blob idx2offset: []const u64, // compressed user gids blob. A blob contains N <= users.len items, // an item is: // len: varint // gid: [varint]varint, // ... and the gid list is delta-compressed. blob: []const u8, }; fn additionalGids( allocator: Allocator, corpus: *const Corpus, ) error{OutOfMemory}!AdditionalGids { var blob = ArrayList(u8).init(allocator); errdefer blob.deinit(); var idx2offset = try allocator.alloc(u64, corpus.users.len); errdefer allocator.free(idx2offset); // zero'th entry is empty, so groupless users can refer to it. try compress.appendUvarint(&blob, 0); var scratch = try allocator.alloc(u32, 256); var scratch_allocated: bool = true; defer if (scratch_allocated) allocator.free(scratch); for (corpus.user2groups) |usergroups, user_idx| { if (usergroups.len == 0) { idx2offset[user_idx] = 0; continue; } idx2offset[user_idx] = blob.items.len; if (scratch.len < usergroups.len) { allocator.free(scratch); scratch_allocated = false; scratch = try allocator.alloc(u32, usergroups.len); scratch_allocated = true; } scratch.len = usergroups.len; const corpusGids = corpus.groups.items(.gid); for (usergroups) |group_idx, i| scratch[i] = corpusGids[group_idx]; compress.deltaCompress(u32, scratch) catch |err| switch (err) { error.NotSorted => unreachable, }; try compress.appendUvarint(&blob, usergroups.len); for (scratch) |gid| try compress.appendUvarint(&blob, gid); } return AdditionalGids{ .idx2offset = idx2offset, .blob = blob.toOwnedSlice(), }; } const UsersSection = struct { // number of users in this section len: u32, // user index -> offset in blob idx2offset: []const u32, blob: []const u8, }; fn usersSection( allocator: Allocator, corpus: *const Corpus, gids: *const AdditionalGids, shells: *const ShellSections, ) error{ OutOfMemory, InvalidRecord, TooMany }!UsersSection { var idx2offset = try allocator.alloc(u32, corpus.users.len); errdefer allocator.free(idx2offset); // as of writing each user takes 12 bytes + blobs + padding, padded to // 8 bytes. 24 is an optimistic lower bound for an average record size. var blob = try ArrayList(u8).initCapacity(allocator, 24 * corpus.users.len); errdefer blob.deinit(); var i: usize = 0; while (i < corpus.users.len) : (i += 1) { // TODO: this is inefficient by calling `.slice()` on every iteration const user = corpus.users.get(i); const user_offset = math.cast(u35, blob.items.len) orelse return error.TooMany; assert(user_offset & 7 == 0); idx2offset[i] = @truncate(u32, user_offset >> 3); try PackedUser.packTo( &blob, user, gids.idx2offset[i], shells.shell2idx, ); try pad.arrayList(&blob, PackedUser.alignment_bits); } return UsersSection{ .len = @intCast(u32, corpus.users.len), .idx2offset = idx2offset, .blob = blob.toOwnedSlice(), }; } const GroupMembers = struct { // group index to it's offset in blob idx2offset: []const u64, // members are delta-varint encoded byte-offsets to the user struct blob: []const u8, }; fn groupMembers( allocator: Allocator, corpus: *const Corpus, user2offset: []const u32, ) error{OutOfMemory}!GroupMembers { var idx2offset = try allocator.alloc(u64, corpus.groups.len); errdefer allocator.free(idx2offset); var blob = ArrayList(u8).init(allocator); errdefer blob.deinit(); // zero'th entry is empty, so empty groups can refer to it try compress.appendUvarint(&blob, 0); var scratch = try ArrayList(u32).initCapacity(allocator, 1024); defer scratch.deinit(); for (corpus.group2users) |members, group_idx| { if (members.len == 0) { idx2offset[group_idx] = 0; continue; } idx2offset[group_idx] = blob.items.len; try scratch.ensureTotalCapacity(members.len); scratch.items.len = members.len; for (members) |user_idx, i| scratch.items[i] = user2offset[user_idx]; compress.deltaCompress(u32, scratch.items) catch |err| switch (err) { error.NotSorted => unreachable, }; try compress.appendUvarint(&blob, members.len); for (scratch.items) |elem| try compress.appendUvarint(&blob, elem); } return GroupMembers{ .idx2offset = idx2offset, .blob = blob.toOwnedSlice(), }; } const GroupsSection = struct { // number of groups in this section len: u32, // group index -> offset in blob idx2offset: []const u32, blob: []const u8, }; fn groupsSection( allocator: Allocator, corpus: *const Corpus, members_offset: []const u64, ) error{ OutOfMemory, InvalidRecord }!GroupsSection { var idx2offset = try allocator.alloc(u32, corpus.groups.len); errdefer allocator.free(idx2offset); var blob = try ArrayList(u8).initCapacity(allocator, 8 * corpus.groups.len); errdefer blob.deinit(); var i: usize = 0; while (i < corpus.groups.len) : (i += 1) { // TODO: this is inefficient; it's calling `.slice()` on every iteration const group = corpus.groups.get(i); const group_offset = @intCast(u32, blob.items.len); assert(group_offset & 7 == 0); idx2offset[i] = @truncate(u32, group_offset >> 3); const group_stored = GroupStored{ .gid = group.gid, .name = group.name, .members_offset = members_offset[i], }; try PackedGroup.packTo(&blob, group_stored); try pad.arrayList(&blob, PackedGroup.alignment_bits); } return GroupsSection{ .len = @intCast(u32, corpus.groups.len), .idx2offset = idx2offset, .blob = blob.toOwnedSlice(), }; } pub fn userGids(self: *const DB, offset: u64) compress.DeltaDecompressionIterator { var vit = compress.varintSliceIteratorMust(self.additional_gids[offset..]); return compress.deltaDecompressionIterator(&vit); } // creates a bdz index using packed_mphf. // hash = bdz_search(packed_mphf, keys[i]); // result[hash] = idx2offset[i]; fn bdzIdx( comptime T: type, allocator: Allocator, packed_mphf: []const u8, keys: []const T, idx2offset: []const u32, ) error{OutOfMemory}![]const u32 { const search_fn = switch (T) { u32 => bdz.search_u32, []const u8 => bdz.search, else => unreachable, }; assert(keys.len <= math.maxInt(u32)); var result = try allocator.alloc(u32, keys.len); errdefer allocator.free(result); for (keys) |key, i| result[search_fn(packed_mphf, key)] = idx2offset[i]; return result; } // nblocks_n returns how many blocks a given number of bytes will take pub fn nblocks_n(comptime T: type, nbytes: usize) T { const B = switch (T) { u8 => u14, u16 => u22, u32 => u38, u64 => u70, else => @compileError("got " ++ @typeName(T) ++ ", only u8, u32 and u64 are supported"), }; const upper = pad.roundUp(B, section_length_bits, @intCast(B, nbytes)); assert(upper & (section_length - 1) == 0); return @truncate(T, upper >> section_length_bits); } // nblocks returns how many blocks a particular slice will take. fn nblocks(comptime T: type, arr: []const u8) T { return nblocks_n(T, arr.len); } fn assertDefinedLayout(comptime T: type) void { return switch (T) { u4, u8, u16, u32, u64 => {}, else => switch (@typeInfo(T)) { .Array => assertDefinedLayout(meta.Elem(T)), .Pointer => |info| assertDefinedLayout(info.child), .Enum => assertDefinedLayout(meta.Tag(T)), .Struct => { if (meta.containerLayout(T) == .Auto) @compileError("layout of " ++ @typeName(T) ++ " is undefined"); for (meta.fields(T)) |field| assertDefinedLayout(field.field_type); }, else => @compileError("unexpected type " ++ @typeName(T)), }, }; } const testing = std.testing; test "DB read/write via iovec" { const allocator = testing.allocator; var corpus = try Corpus.testCorpus(allocator); defer corpus.deinit(); var errc = ErrCtx{}; var db = try DB.fromCorpus(allocator, &corpus, &errc); defer db.deinit(allocator); const fd = try os.memfd_create("test_turbonss_db", 0); defer os.close(fd); const len = try os.writev(fd, db.iov().constSlice()); const buf = try os.mmap(null, len, os.PROT.READ, os.MAP.SHARED, fd, 0); const db2 = try fromBytes(buf); try testing.expectEqual(corpus.groups.len, db.header.num_groups); try testing.expectEqual(corpus.users.len, db.header.num_users); try testing.expectEqual(db.header.num_groups, db2.header.num_groups); try testing.expectEqual(db.header.num_users, db2.header.num_users); const num_groups = db2.header.num_groups; const num_users = db2.header.num_users; try testing.expectEqualSlices(u32, db.idx_gid2group, db2.idx_gid2group[0..num_groups]); try testing.expectEqualSlices(u32, db.idx_uid2user, db2.idx_uid2user[0..num_users]); try testing.expectEqualStrings("", errc.unwrap().constSlice()); } test "DB getgrnam/getgrgid" { var corpus = try Corpus.testCorpus(testing.allocator); defer corpus.deinit(); var errc = ErrCtx{}; var db = try DB.fromCorpus(testing.allocator, &corpus, &errc); defer db.deinit(testing.allocator); var buf = try testing.allocator.alloc(u8, db.getgrBufsize()); defer testing.allocator.free(buf); { try testing.expectEqual(try db.getgrnam("doesnotexist", buf, false), null); const all = (try db.getgrnam("all", buf, false)).?; try testing.expectEqual(all.gr_gid, 9999); try testing.expectEqualStrings(all.gr_name[0..4], "all\x00"); const members = all.gr_mem; try testing.expectEqualStrings(mem.sliceTo(members[0].?, 0), "Name" ** 8); try testing.expectEqualStrings(mem.sliceTo(members[1].?, 0), "root"); try testing.expectEqualStrings(mem.sliceTo(members[2].?, 0), "svc-bar"); try testing.expectEqualStrings(mem.sliceTo(members[3].?, 0), "vidmantas"); try testing.expectEqual(members[4], null); } { const all = (try db.getgrnam("all", buf, true)).?; try testing.expectEqual(all.gr_gid, 9999); try testing.expectEqualStrings(all.gr_name[0..4], "all\x00"); try testing.expectEqual(all.gr_mem[0], null); } { try testing.expectEqual(try db.getgrgid(42, buf, false), null); const all = (try db.getgrgid(9999, buf, false)).?; try testing.expectEqual(all.gr_gid, 9999); try testing.expectEqualStrings(all.gr_name[0..3], "all"); } try testing.expectEqualStrings("", errc.unwrap().constSlice()); _ = try db.getgrnam("all", buf, false); buf.len -= 1; try testing.expectError(error.BufferTooSmall, db.getgrnam("all", buf, false)); } test "DB getpwnam/getpwuid" { var corpus = try Corpus.testCorpus(testing.allocator); defer corpus.deinit(); var errc = ErrCtx{}; var db = try DB.fromCorpus(testing.allocator, &corpus, &errc); defer db.deinit(testing.allocator); var buf = try testing.allocator.alloc(u8, db.getpwBufsize()); defer testing.allocator.free(buf); { try testing.expectEqual(try db.getpwnam("doesnotexist", buf), null); const vidmantas = (try db.getpwnam("vidmantas", buf)).?; try testing.expectEqual(vidmantas.pw_uid, 128); try testing.expectEqual(vidmantas.pw_gid, 128); try testing.expectEqualStrings(vidmantas.pw_name[0..10], "vidmantas\x00"); try testing.expectEqualStrings(vidmantas.pw_gecos[0..20], "Vidmantas Kaminskas\x00"); try testing.expectEqualStrings(vidmantas.pw_dir[0..16], "/home/vidmantas\x00"); } { try testing.expectEqual(try db.getpwuid(123456, buf), null); const vidmantas = (try db.getpwuid(128, buf)).?; try testing.expectEqual(vidmantas.pw_uid, 128); try testing.expectEqual(vidmantas.pw_gid, 128); try testing.expectEqualStrings(vidmantas.pw_name[0..10], "vidmantas\x00"); } try testing.expectEqualStrings("", errc.unwrap().constSlice()); const long = try db.getpwnam("Name" ** 8, buf); try testing.expectEqualStrings(long.?.pw_name[0..33], "Name" ** 8 ++ "\x00"); buf.len -= 1; try testing.expectError(error.BufferTooSmall, db.getpwnam("Name" ** 8, buf)); } test "DB additionalGids" { const allocator = testing.allocator; var corpus = try Corpus.testCorpus(allocator); defer corpus.deinit(); var additional_gids = try additionalGids(allocator, &corpus); defer allocator.free(additional_gids.idx2offset); defer allocator.free(additional_gids.blob); var user_idx: usize = 0; while (user_idx < corpus.users.len) : (user_idx += 1) { const groups = corpus.user2groups[user_idx]; const offset = additional_gids.idx2offset[user_idx]; if (groups.len == 0) { try testing.expect(offset == 0); continue; } var vit = try compress.varintSliceIterator(additional_gids.blob[offset..]); var it = compress.deltaDecompressionIterator(&vit); try testing.expectEqual(it.remaining(), groups.len); var i: u64 = 0; const corpusGids = corpus.groups.items(.gid); while (try it.next()) |gid| : (i += 1) { try testing.expectEqual(gid, corpusGids[groups[i]]); } try testing.expectEqual(i, groups.len); } } test "DB pack gids" { const allocator = testing.allocator; var corpus = try Corpus.testCorpus(allocator); defer corpus.deinit(); const cmph_gid = try cmph.packU32(allocator, corpus.groups.items(.gid)); defer allocator.free(cmph_gid); const k1 = bdz.search_u32(cmph_gid, 0); const k2 = bdz.search_u32(cmph_gid, 128); const k3 = bdz.search_u32(cmph_gid, 9999); const k4 = bdz.search_u32(cmph_gid, 100000); var hashes = &[_]u32{ k1, k2, k3, k4 }; sort.sort(u32, hashes, {}, comptime sort.asc(u32)); for (hashes) |hash, i| try testing.expectEqual(i, hash); } const hash_offsets = &[_]u32{ 0, 10, 20, 30 }; fn expectUsedHashes(allocator: Allocator, arr: []const u32) !void { var used = AutoHashMap(u32, void).init(allocator); defer used.deinit(); for (arr) |elem| try used.putNoClobber(elem, {}); for (hash_offsets) |item| try testing.expect(used.get(item) != null); } test "DB bdzIdx on u32" { const keys = [_]u32{ 42, 1, 2, 3 }; const mphf = try cmph.packU32(testing.allocator, keys[0..]); defer testing.allocator.free(mphf); var result = try bdzIdx(u32, testing.allocator, mphf, keys[0..], hash_offsets); defer testing.allocator.free(result); try expectUsedHashes(testing.allocator, result); } test "DB bdzIdx on str" { const keys = [_][]const u8{ "42", "1", "2", "3" }; const mphf = try cmph.packStr(testing.allocator, keys[0..]); defer testing.allocator.free(mphf); var result = try bdzIdx([]const u8, testing.allocator, mphf, keys[0..], hash_offsets); defer testing.allocator.free(result); try expectUsedHashes(testing.allocator, result); } test "DB nblocks" { const tests = .{ .{ 0, &[_]u8{} }, .{ 1, &[_]u8{ 1, 2, 42 } }, .{ 1, &[_]u8{1} ** 63 }, .{ 1, &[_]u8{1} ** 64 }, .{ 2, &[_]u8{1} ** 65 }, .{ 255, &[_]u8{1} ** (255 * 64) }, }; inline for (tests) |tt| { try testing.expectEqual(nblocks(u8, tt[1]), tt[0]); try testing.expectEqual(nblocks(u32, tt[1]), tt[0]); try testing.expectEqual(nblocks(u64, tt[1]), tt[0]); } }