motiejus
/
turbonss
1
Fork 0
Code Releases Activity

DB.header is becoming a pointer 

Also, move User, PackedUser, Group, PackedGroup to better-fit places.
main
Motiejus Jakštys 2022-03-25 14:31:56 +02:00 committed by Motiejus Jakštys
parent 347f0a1392
commit fad7c9fbaf
10 changed files with 682 additions and 657 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/Corpus.zig
View File

@ -9,8 +9,8 @@ const StringHashMap = std.StringHashMap;
const MultiArrayList = std.MultiArrayList;
const ArrayListUnmanaged = std.ArrayListUnmanaged;
const User = @import("user.zig").User;
const Group = @import("group.zig").Group;
const User = @import("User.zig");
const Group = @import("Group.zig");
pub const Corpus = @This();
@ -152,7 +152,7 @@ fn testUser(name: []const u8) User {
}
const testing = std.testing;
const someMembers = @import("group.zig").someMembers;
const someMembers = @import("Group.zig").someMembers;
test "users compare function" {
const a = testUser("a");

165
lib/DB.zig
View File

@ -13,14 +13,15 @@ const BoundedArray = std.BoundedArray;
const Corpus = @import("Corpus.zig");
const pad = @import("padding.zig");
const compress = @import("compress.zig");
const PackedUser = @import("user.zig").PackedUser;
const User = @import("user.zig").User;
const Group = @import("group.zig").Group;
const GroupStored = @import("group.zig").GroupStored;
const PackedGroup = @import("group.zig").PackedGroup;
const PackedUser = @import("PackedUser.zig");
const User = @import("User.zig");
const Group = @import("Group.zig");
const PackedGroup = @import("PackedGroup.zig");
const GroupStored = PackedGroup.GroupStored;
const ShellSections = @import("shell.zig").ShellWriter.ShellSections;
const ShellReader = @import("shell.zig").ShellReader;
const ShellWriter = @import("shell.zig").ShellWriter;
const InvalidHeader = @import("header.zig").Invalid;
const Header = @import("header.zig").Header;
const max_shells = @import("shell.zig").max_shells;
const section_length_bits = @import("header.zig").section_length_bits;
@ -32,7 +33,7 @@ const zeroes = &[_]u8{0} ** section_length;
const DB = @This();
// All sections, as they end up in the DB. Order is important.
header: Header,
header: *const Header,
bdz_gid: []const u8,
bdz_groupname: []const u8,
bdz_uid: []const u8,
@ -57,13 +58,13 @@ pub fn fromCorpus(
const uids = corpus.users.items(.uid);
const unames = corpus.users.items(.name);
var bdz_gid = try cmph.packU32(allocator, gids);
const bdz_gid = try cmph.packU32(allocator, gids);
errdefer allocator.free(bdz_gid);
var bdz_groupname = try cmph.packStr(allocator, gnames);
const bdz_groupname = try cmph.packStr(allocator, gnames);
errdefer allocator.free(bdz_groupname);
var bdz_uid = try cmph.packU32(allocator, uids);
const bdz_uid = try cmph.packU32(allocator, uids);
errdefer allocator.free(bdz_uid);
const bdz_username = try cmph.packStr(allocator, unames);
@ -72,35 +73,38 @@ pub fn fromCorpus(
var shell = try shellSections(allocator, corpus);
defer shell.deinit();
var additional_gids = try additionalGids(allocator, corpus);
const additional_gids = try additionalGids(allocator, corpus);
errdefer allocator.free(additional_gids.blob);
defer allocator.free(additional_gids.idx2offset);
var users = try usersSection(allocator, corpus, &additional_gids, &shell);
allocator.free(additional_gids.idx2offset);
const users = try usersSection(allocator, corpus, &additional_gids, &shell);
errdefer allocator.free(users.blob);
defer allocator.free(users.idx2offset);
var groupmembers = try groupMembers(allocator, corpus, users.idx2offset);
const groupmembers = try groupMembers(allocator, corpus, users.idx2offset);
errdefer allocator.free(groupmembers.blob);
defer allocator.free(groupmembers.idx2offset);
var groups = try groupsSection(allocator, corpus, groupmembers.idx2offset);
allocator.free(groupmembers.idx2offset);
const groups = try groupsSection(allocator, corpus, groupmembers.idx2offset);
errdefer allocator.free(groups.blob);
defer allocator.free(groups.idx2offset);
var idx_gid2group = try bdzIdx(u32, allocator, bdz_gid, gids, groups.idx2offset);
const idx_gid2group = try bdzIdx(u32, allocator, bdz_gid, gids, groups.idx2offset);
errdefer allocator.free(idx_gid2group);
var idx_groupname2group = try bdzIdx([]const u8, allocator, bdz_groupname, gnames, groups.idx2offset);
allocator.free(groups.idx2offset);
const idx_groupname2group = try bdzIdx([]const u8, allocator, bdz_groupname, gnames, groups.idx2offset);
errdefer allocator.free(idx_groupname2group);
var idx_uid2user = try bdzIdx(u32, allocator, bdz_uid, uids, users.idx2offset);
const idx_uid2user = try bdzIdx(u32, allocator, bdz_uid, uids, users.idx2offset);
errdefer allocator.free(idx_uid2user);
var idx_name2user = try bdzIdx([]const u8, allocator, bdz_username, unames, users.idx2offset);
allocator.free(users.idx2offset);
const idx_name2user = try bdzIdx([]const u8, allocator, bdz_username, unames, users.idx2offset);
errdefer allocator.free(idx_name2user);
const header = Header{
const header = try allocator.create(Header);
errdefer allocator.destroy(header);
header.* = Header{
.nblocks_shell_blob = nblocks(u8, shell.blob.constSlice()),
.num_shells = shell.len,
.num_groups = groups.len,
@ -134,6 +138,23 @@ pub fn fromCorpus(
};
}
pub fn deinit(self: *DB, allocator: Allocator) void {
allocator.destroy(self.header);
allocator.free(self.bdz_gid);
allocator.free(self.bdz_groupname);
allocator.free(self.bdz_uid);
allocator.free(self.bdz_username);
allocator.free(self.idx_gid2group);
allocator.free(self.idx_groupname2group);
allocator.free(self.idx_uid2user);
allocator.free(self.idx_name2user);
allocator.free(self.groups);
allocator.free(self.users);
allocator.free(self.groupmembers);
allocator.free(self.additional_gids);
self.* = undefined;
}
const DB_fields = meta.fields(DB);
pub fn iov(self: *const DB) BoundedArray(os.iovec_const, DB_fields.len * 2) {
var result = BoundedArray(os.iovec_const, DB_fields.len * 2).init(0) catch unreachable;
@ -141,7 +162,7 @@ pub fn iov(self: *const DB) BoundedArray(os.iovec_const, DB_fields.len * 2) {
comptime assertDefinedLayout(field.field_type);
const value = @field(self, field.name);
const bytes: []const u8 = switch (@TypeOf(value)) {
Header => mem.asBytes(&value),
*const Header => mem.asBytes(value),
else => mem.sliceAsBytes(value),
};
result.appendAssumeCapacity(os.iovec_const{
@ -159,58 +180,44 @@ pub fn iov(self: *const DB) BoundedArray(os.iovec_const, DB_fields.len * 2) {
return result;
}
pub fn fromBytes(buf: []const u8) Header.Invalid!DB {
const header = try Header.fromBytes(buf);
pub fn fromBytes(buf: []align(8) const u8) InvalidHeader!DB {
const header = try Header.fromBytes(buf[0..@sizeOf(Header)]);
// At first the tuple below had field names too, but moved it to comments,
// because it segfaulted. https://github.com/ziglang/zig/issues/3915 and
// https://paste.sr.ht/~motiejus/2830736e796801517c1fa8639be6615cd56ada27
const lengths = .{
.{ "bdz_gid", header.nblocks_bdz_gid },
.{ "bdz_groupname", header.nblocks_bdz_groupname },
.{ "bdz_uid", header.nblocks_bdz_uid },
.{ "bdz_username", header.nblocks_bdz_username },
.{ "idx_gid2group", nblocks_n(header.num_groups * 4) },
.{ "idx_groupname2group", nblocks_n(header.num_groups * 4) },
.{ "idx_uid2user", nblocks_n(header.num_users * 4) },
.{ "idx_name2user", nblocks_n(header.num_users * 4) },
.{ "shell_index", nblocks_n(header.num_shells * 2) },
.{ "shell_blob", header.nblocks_shell_blob },
.{ "groups", header.nblocks_groups },
.{ "users", header.nblocks_users },
.{ "groupmembers", header.nblocks_groupmembers },
.{ "additional_gids", header.nblocks_additional_gids },
header.nblocks_bdz_gid, // bdz_gid
header.nblocks_bdz_groupname, // bdz_groupname
header.nblocks_bdz_uid, // bdz_uid
header.nblocks_bdz_username, // bdz_username
nblocks_n(u32, header.num_groups * 4), // idx_gid2group
nblocks_n(u32, header.num_groups * 4), // idx_groupname2group
nblocks_n(u32, header.num_users * 4), // idx_uid2user
nblocks_n(u32, header.num_users * 4), // idx_name2user
nblocks_n(u16, header.num_shells * 2), // shell_index
header.nblocks_shell_blob, // shell_blob
header.nblocks_groups, // groups
header.nblocks_users, // users
header.nblocks_groupmembers, // groupmembers
header.nblocks_additional_gids, // additional_gids
};
var result: DB = undefined;
result.header = header;
var offset = comptime nblocks_n(usize, @sizeOf(Header));
comptime assert(DB_fields[0].name == "header");
var offset = comptime nblocks_n(u64, @sizeOf(Header));
comptime assert(mem.eql(u8, DB_fields[0].name, "header"));
inline for (DB_fields[1..]) |field, i| {
assert(lengths[i][0] == field.name);
const start = offset << 6;
const end = (offset + lengths[i][1]) << 6;
const value = mem.bytesAsValue(field.field_type, buf[start..end]);
const start = offset << section_length_bits;
const end = (offset + lengths[i]) << section_length_bits;
const slice_type = meta.Child(field.field_type);
const value = mem.bytesAsSlice(slice_type, buf[start..end]);
@field(result, field.name) = value;
offset += lengths[i][1];
offset += lengths[i];
}
return result;
}
pub fn deinit(self: *DB, allocator: Allocator) void {
allocator.free(self.bdz_gid);
allocator.free(self.bdz_groupname);
allocator.free(self.bdz_uid);
allocator.free(self.bdz_username);
allocator.free(self.idx_gid2group);
allocator.free(self.idx_groupname2group);
allocator.free(self.idx_uid2user);
allocator.free(self.idx_name2user);
allocator.free(self.groups);
allocator.free(self.users);
allocator.free(self.groupmembers);
allocator.free(self.additional_gids);
self.* = undefined;
}
fn shellSections(
allocator: Allocator,
corpus: *const Corpus,
@ -414,12 +421,10 @@ fn bdzIdx(
keys: []const T,
idx2offset: []const u32,
) error{OutOfMemory}![]const u32 {
const search_fn = comptime blk: {
switch (T) {
u32 => break :blk bdz.search_u32,
[]const u8 => break :blk bdz.search,
else => unreachable,
}
const search_fn = switch (T) {
u32 => bdz.search_u32,
[]const u8 => bdz.search,
else => unreachable,
};
assert(keys.len <= math.maxInt(u32));
var result = try allocator.alloc(u32, keys.len);
@ -432,13 +437,14 @@ fn bdzIdx(
fn nblocks_n(comptime T: type, nbytes: usize) T {
const B = switch (T) {
u8 => u14,
u16 => u22,
u32 => u38,
u64 => u70,
else => @compileError("only u8, u32 and u64 are supported"),
else => @compileError("got " ++ @typeName(T) ++ ", only u8, u32 and u64 are supported"),
};
const upper = pad.roundUp(B, section_length_bits, @intCast(B, nbytes));
assert(upper & (section_length - 1) == 0);
return @truncate(T, upper >> 6);
return @truncate(T, upper >> section_length_bits);
}
// nblocks returns how many blocks a particular slice will take.
@ -450,7 +456,8 @@ fn assertDefinedLayout(comptime T: type) void {
return switch (T) {
u8, u16, u32, u64 => {},
else => switch (@typeInfo(T)) {
.Array, .Pointer => assertDefinedLayout(meta.Elem(T)),
.Array => assertDefinedLayout(meta.Elem(T)),
.Pointer => |info| assertDefinedLayout(info.child),
.Enum => assertDefinedLayout(meta.Tag(T)),
.Struct => {
if (meta.containerLayout(T) == .Auto)
@ -505,8 +512,18 @@ test "test groups, group members and users" {
const fd = try os.memfd_create("test_turbonss_db", 0);
defer os.close(fd);
const written = try os.writev(fd, db.iov().constSlice());
try testing.expect(written != 0);
const len = try os.writev(fd, db.iov().constSlice());
const buf = try os.mmap(null, len, os.PROT.READ, os.MAP.SHARED, fd, 0);
const db2 = try fromBytes(buf);
try testing.expectEqual(corpus.groups.len, db.header.num_groups);
try testing.expectEqual(corpus.users.len, db.header.num_users);
try testing.expectEqual(db.header.num_groups, db2.header.num_groups);
try testing.expectEqual(db.header.num_users, db2.header.num_users);
const num_groups = db2.header.num_groups;
const num_users = db2.header.num_users;
try testing.expectEqualSlices(u32, db.idx_gid2group, db2.idx_gid2group[0..num_groups]);
try testing.expectEqualSlices(u32, db.idx_uid2user, db2.idx_uid2user[0..num_users]);
}
test "additionalGids" {

61
lib/Group.zig Normal file
View File

@ -0,0 +1,61 @@
const std = @import("std");
const mem = std.mem;
const Allocator = mem.Allocator;
const BufSet = std.BufSet;
const Group = @This();
gid: u32,
name: []const u8,
members: BufSet,
pub fn clone(self: *const Group, allocator: Allocator) Allocator.Error!Group {
var name = try allocator.dupe(u8, self.name);
return Group{
.gid = self.gid,
.name = name,
.members = try self.members.cloneWithAllocator(allocator),
};
}
pub fn deinit(self: *Group, allocator: Allocator) void {
allocator.free(self.name);
self.members.deinit();
self.* = undefined;
}
// someMembers constructs a bufset from an allocator and a list of strings.
pub fn someMembers(
allocator: Allocator,
members: []const []const u8,
) Allocator.Error!BufSet {
var bufset = BufSet.init(allocator);
errdefer bufset.deinit();
for (members) |member|
try bufset.insert(member);
return bufset;
}
const testing = std.testing;
test "Group.clone" {
var allocator = testing.allocator;
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
var members = BufSet.init(allocator);
try members.insert("member1");
try members.insert("member2");
defer members.deinit();
var cloned = try members.cloneWithAllocator(arena.allocator());
cloned.remove("member1");
try cloned.insert("member4");
try testing.expect(members.contains("member1"));
try testing.expect(!members.contains("member4"));
try testing.expect(!cloned.contains("member1"));
try testing.expect(cloned.contains("member4"));
}

149
lib/PackedGroup.zig Normal file
View File

@ -0,0 +1,149 @@
const std = @import("std");
const pad = @import("padding.zig");
const validate = @import("validate.zig");
const compress = @import("compress.zig");
const InvalidRecord = validate.InvalidRecord;
const mem = std.mem;
const Allocator = mem.Allocator;
const ArrayList = std.ArrayList;
const BufSet = std.BufSet;
const PackedGroup = @This();
pub const GroupStored = struct {
gid: u32,
name: []const u8,
members_offset: u64,
};
pub const alignment_bits = 3;
const Inner = packed struct {
gid: u32,
padding: u3 = 0,
groupname_len: u5,
pub fn groupnameLen(self: *const Inner) usize {
return @as(usize, self.groupname_len) + 1;
}
};
inner: *const Inner,
groupdata: []const u8,
members_offset: u64,
pub const Entry = struct {
group: PackedGroup,
next: ?[]const u8,
};
pub fn fromBytes(bytes: []const u8) Entry {
const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]);
const start_blob = @sizeOf(Inner);
const end_strings = @sizeOf(Inner) + inner.groupnameLen();
const members_offset = compress.uvarint(bytes[end_strings..]) catch |err| switch (err) {
error.Overflow => unreachable,
};
const end_blob = end_strings + members_offset.bytes_read;
const next_start = pad.roundUp(usize, alignment_bits, end_blob);
var next: ?[]const u8 = null;
if (next_start < bytes.len)
next = bytes[next_start..];
return Entry{
.group = PackedGroup{
.inner = inner,
.groupdata = bytes[start_blob..end_strings],
.members_offset = members_offset.value,
},
.next = next,
};
}
fn validateUtf8(s: []const u8) InvalidRecord!void {
if (!std.unicode.utf8ValidateSlice(s))
return error.InvalidRecord;
}
pub const Iterator = struct {
section: ?[]const u8,
pub fn next(it: *Iterator) ?PackedGroup {
if (it.section) |section| {
const entry = fromBytes(section);
it.section = entry.next;
return entry.group;
}
return null;
}
};
pub fn iterator(section: []const u8) Iterator {
return Iterator{ .section = section };
}
pub fn gid(self: *const PackedGroup) u32 {
return self.inner.gid;
}
pub fn membersOffset(self: *const PackedGroup) u64 {
return self.members_offset;
}
pub fn name(self: *const PackedGroup) []const u8 {
return self.groupdata;
}
pub fn packTo(
arr: *ArrayList(u8),
group: GroupStored,
) error{ InvalidRecord, OutOfMemory }!void {
std.debug.assert(arr.items.len & 7 == 0);
try validate.utf8(group.name);
const len = try validate.downCast(u5, group.name.len - 1);
const inner = Inner{ .gid = group.gid, .groupname_len = len };
try arr.*.appendSlice(mem.asBytes(&inner));
try arr.*.appendSlice(group.name);
try compress.appendUvarint(arr, group.members_offset);
}
const testing = std.testing;
test "PackedGroup alignment" {
try testing.expectEqual(@sizeOf(PackedGroup) * 8, @bitSizeOf(PackedGroup));
}
test "construct PackedGroups" {
var buf = ArrayList(u8).init(testing.allocator);
defer buf.deinit();
const groups = [_]GroupStored{
GroupStored{
.gid = 1000,
.name = "sudo",
.members_offset = 1,
},
GroupStored{
.gid = std.math.maxInt(u32),
.name = "Name" ** 8, // 32
.members_offset = std.math.maxInt(u64),
},
};
for (groups) |group| {
try PackedGroup.packTo(&buf, group);
try pad.arrayList(&buf, PackedGroup.alignment_bits);
}
var i: u29 = 0;
var it = PackedGroup.iterator(buf.items);
while (it.next()) |group| : (i += 1) {
try testing.expectEqual(groups[i].gid, group.gid());
try testing.expectEqualStrings(groups[i].name, group.name());
try testing.expectEqual(groups[i].members_offset, group.membersOffset());
}
try testing.expectEqual(groups.len, i);
}

288
lib/PackedUser.zig Normal file
View File

@ -0,0 +1,288 @@
const std = @import("std");
const assert = std.debug.assert;
const mem = std.mem;
const math = std.math;
const Allocator = mem.Allocator;
const ArrayList = std.ArrayList;
const StringHashMap = std.StringHashMap;
const pad = @import("padding.zig");
const validate = @import("validate.zig");
const compress = @import("compress.zig");
const shellImport = @import("shell.zig");
const InvalidRecord = validate.InvalidRecord;
const User = @import("User.zig");
const PackedUser = @This();
pub const alignment_bits = 3;
const Inner = packed struct {
uid: u32,
gid: u32,
shell_len_or_idx: u8,
shell_here: bool,
name_is_a_suffix: bool,
home_len: u6,
name_len: u5,
gecos_len: u11,
fn homeLen(self: *const Inner) usize {
return @as(u32, self.home_len) + 1;
}
fn nameStart(self: *const Inner) usize {
const name_len = self.nameLen();
if (self.name_is_a_suffix) {
return self.homeLen() - name_len;
} else return self.homeLen();
}
fn nameLen(self: *const Inner) usize {
return @as(u32, self.name_len) + 1;
}
fn gecosStart(self: *const Inner) usize {
if (self.name_is_a_suffix) {
return self.homeLen();
} else return self.homeLen() + self.nameLen();
}
fn gecosLen(self: *const Inner) usize {
return self.gecos_len;
}
fn maybeShellStart(self: *const Inner) usize {
assert(self.shell_here);
return self.gecosStart() + self.gecosLen();
}
fn shellLen(self: *const Inner) usize {
return @as(u32, self.shell_len_or_idx) + 1;
}
// stringLength returns the length of the blob storing string values.
fn stringLength(self: *const Inner) usize {
var result: usize = self.homeLen() + self.gecosLen();
if (!self.name_is_a_suffix)
result += self.nameLen();
if (self.shell_here)
result += self.shellLen();
return result;
}
};
// PackedUser does not allocate; it re-interprets the "bytes" blob
// field. Both of those fields are pointers to "our representation" of
// that field.
inner: *const Inner,
bytes: []const u8,
additional_gids_offset: u64,
pub const Entry = struct {
user: PackedUser,
next: ?[]const u8,
};
// TODO(motiejus) provide a way to return an entry without decoding the
// additional_gids_offset:
// - will not return the 'next' slice.
// - cannot throw an Overflow error.
pub fn fromBytes(bytes: []const u8) Entry {
const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]);
const start_blob = @sizeOf(Inner);
const end_strings = start_blob + inner.stringLength();
const gids_offset = compress.uvarint(bytes[end_strings..]) catch |err| switch (err) {
error.Overflow => unreachable,
};
const end_blob = end_strings + gids_offset.bytes_read;
const nextStart = pad.roundUp(usize, alignment_bits, end_blob);
var next: ?[]const u8 = null;
if (nextStart < bytes.len)
next = bytes[nextStart..];
return Entry{
.user = PackedUser{
.inner = inner,
.bytes = bytes[start_blob..end_blob],
.additional_gids_offset = gids_offset.value,
},
.next = next,
};
}
pub const Iterator = struct {
section: ?[]const u8,
shell_reader: shellImport.ShellReader,
pub fn next(it: *Iterator) ?PackedUser {
if (it.section) |section| {
const entry = PackedUser.fromBytes(section);
it.section = entry.next;
return entry.user;
}
return null;
}
};
pub fn iterator(section: []const u8, shell_reader: shellImport.ShellReader) Iterator {
return Iterator{ .section = section, .shell_reader = shell_reader };
}
// packTo packs the User record and copies it to the given byte slice.
// The slice must have at least maxRecordSize() bytes available. The
// slice is passed as a pointer, so it can be mutated.
pub fn packTo(
arr: *ArrayList(u8),
user: User,
additional_gids_offset: u64,
idxFn: StringHashMap(u8),
) error{ InvalidRecord, OutOfMemory }!void {
std.debug.assert(arr.items.len & 7 == 0);
// function arguments are consts. We need to mutate the underlying
// slice, so passing it via pointer instead.
const home_len = try validate.downCast(u6, user.home.len - 1);
const name_len = try validate.downCast(u5, user.name.len - 1);
const shell_len = try validate.downCast(u8, user.shell.len - 1);
const gecos_len = try validate.downCast(u8, user.gecos.len);
try validate.utf8(user.home);
try validate.utf8(user.name);
try validate.utf8(user.shell);
try validate.utf8(user.gecos);
const inner = Inner{
.uid = user.uid,
.gid = user.gid,
.shell_here = idxFn.get(user.shell) == null,
.shell_len_or_idx = idxFn.get(user.shell) orelse shell_len,
.home_len = home_len,
.name_is_a_suffix = mem.endsWith(u8, user.home, user.name),
.name_len = name_len,
.gecos_len = gecos_len,
};
const innerBytes = mem.asBytes(&inner);
try arr.*.appendSlice(innerBytes[0..@sizeOf(Inner)]);
try arr.*.appendSlice(user.home);
if (!inner.name_is_a_suffix)
try arr.*.appendSlice(user.name);
try arr.*.appendSlice(user.gecos);
if (inner.shell_here)
try arr.*.appendSlice(user.shell);
try compress.appendUvarint(arr, additional_gids_offset);
}
pub fn uid(self: PackedUser) u32 {
return self.inner.uid;
}
pub fn gid(self: PackedUser) u32 {
return self.inner.gid;
}
pub fn additionalGidsOffset(self: PackedUser) u64 {
return self.additional_gids_offset;
}
pub fn home(self: PackedUser) []const u8 {
return self.bytes[0..self.inner.homeLen()];
}
pub fn name(self: PackedUser) []const u8 {
const name_pos = self.inner.nameStart();
const name_len = self.inner.nameLen();
return self.bytes[name_pos .. name_pos + name_len];
}
pub fn gecos(self: PackedUser) []const u8 {
const gecos_pos = self.inner.gecosStart();
const gecos_len = self.inner.gecosLen();
return self.bytes[gecos_pos .. gecos_pos + gecos_len];
}
pub fn shell(self: PackedUser, shell_reader: shellImport.ShellReader) []const u8 {
if (self.inner.shell_here) {
const shell_pos = self.inner.maybeShellStart();
const shell_len = self.inner.shellLen();
return self.bytes[shell_pos .. shell_pos + shell_len];
}
return shell_reader.get(self.inner.shell_len_or_idx);
}
const testing = std.testing;
test "PackedUser internal and external alignment" {
try testing.expectEqual(
@sizeOf(PackedUser.Inner) * 8,
@bitSizeOf(PackedUser.Inner),
);
}
fn testShellIndex(allocator: Allocator) StringHashMap(u8) {
var result = StringHashMap(u8).init(allocator);
result.put("/bin/bash", 0) catch unreachable;
result.put("/bin/zsh", 1) catch unreachable;
return result;
}
const test_shell_reader = shellImport.ShellReader{
.blob = "/bin/bash/bin/zsh",
.index = &[_]u16{ 0, 9, 17 },
};
test "construct PackedUser section" {
var buf = ArrayList(u8).init(testing.allocator);
defer buf.deinit();
const users = [_]User{ User{
.uid = 1000,
.gid = 1000,
.name = "vidmantas",
.gecos = "Vidmantas Kaminskas",
.home = "/home/vidmantas",
.shell = "/bin/bash",
}, User{
.uid = 1001,
.gid = 1001,
.name = "svc-foo",
.gecos = "Service Account",
.home = "/home/service1",
.shell = "/usr/bin/nologin",
}, User{
.uid = 0,
.gid = math.maxInt(u32),
.name = "Name" ** 8,
.gecos = "Gecos" ** 51,
.home = "Home" ** 16,
.shell = "She.LllL" ** 32,
}, User{
.uid = 1002,
.gid = 1002,
.name = "svc-bar",
.gecos = "",
.home = "/",
.shell = "/bin/zsh",
} };
var shellIndex = testShellIndex(testing.allocator);
const additional_gids = math.maxInt(u64);
defer shellIndex.deinit();
for (users) |user| {
try PackedUser.packTo(&buf, user, additional_gids, shellIndex);
try pad.arrayList(&buf, PackedUser.alignment_bits);
}
var i: u29 = 0;
var it1 = PackedUser.iterator(buf.items, test_shell_reader);
while (it1.next()) |user| : (i += 1) {
try testing.expectEqual(users[i].uid, user.uid());
try testing.expectEqual(users[i].gid, user.gid());
try testing.expectEqual(user.additionalGidsOffset(), additional_gids);
try testing.expectEqualStrings(users[i].name, user.name());
try testing.expectEqualStrings(users[i].gecos, user.gecos());
try testing.expectEqualStrings(users[i].home, user.home());
try testing.expectEqualStrings(users[i].shell, user.shell(test_shell_reader));
}
try testing.expectEqual(users.len, i);
}

67
lib/User.zig Normal file
View File

@ -0,0 +1,67 @@
const std = @import("std");
const mem = std.mem;
const Allocator = mem.Allocator;
const User = @This();
uid: u32,
gid: u32,
name: []const u8,
gecos: []const u8,
home: []const u8,
shell: []const u8,
// deep-clones a User record with a given Allocator.
pub fn clone(
self: *const User,
allocator: Allocator,
) error{OutOfMemory}!User {
const stringdata = try allocator.alloc(u8, self.strlen());
const gecos_start = self.name.len;
const home_start = gecos_start + self.gecos.len;
const shell_start = home_start + self.home.len;
mem.copy(u8, stringdata[0..self.name.len], self.name);
mem.copy(u8, stringdata[gecos_start..], self.gecos);
mem.copy(u8, stringdata[home_start..], self.home);
mem.copy(u8, stringdata[shell_start..], self.shell);
return User{
.uid = self.uid,
.gid = self.gid,
.name = stringdata[0..self.name.len],
.gecos = stringdata[gecos_start .. gecos_start + self.gecos.len],
.home = stringdata[home_start .. home_start + self.home.len],
.shell = stringdata[shell_start .. shell_start + self.shell.len],
};
}
fn strlen(self: *const User) usize {
return self.name.len +
self.gecos.len +
self.home.len +
self.shell.len;
}
pub fn deinit(self: *User, allocator: Allocator) void {
const slice = self.home.ptr[0..self.strlen()];
allocator.free(slice);
self.* = undefined;
}
const testing = std.testing;
test "User.clone" {
var allocator = testing.allocator;
const user = User{
.uid = 1000,
.gid = 1000,
.name = "vidmantas",
.gecos = "Vidmantas Kaminskas",
.home = "/home/vidmantas",
.shell = "/bin/bash",
};
var user2 = try user.clone(allocator);
defer user2.deinit(allocator);
try testing.expectEqualStrings(user.shell, "/bin/bash");
}

201
lib/group.zig
View File

@ -1,201 +0,0 @@
const std = @import("std");
const pad = @import("padding.zig");
const validate = @import("validate.zig");
const compress = @import("compress.zig");
const InvalidRecord = validate.InvalidRecord;
const mem = std.mem;
const Allocator = mem.Allocator;
const ArrayList = std.ArrayList;
const BufSet = std.BufSet;
pub const Group = struct {
gid: u32,
name: []const u8,
members: BufSet,
pub fn clone(self: *const Group, allocator: Allocator) Allocator.Error!Group {
var name = try allocator.dupe(u8, self.name);
return Group{
.gid = self.gid,
.name = name,
.members = try self.members.cloneWithAllocator(allocator),
};
}
pub fn deinit(self: *Group, allocator: Allocator) void {
allocator.free(self.name);
self.members.deinit();
self.* = undefined;
}
};
pub const GroupStored = struct {
gid: u32,
name: []const u8,
members_offset: u64,
};
pub const PackedGroup = struct {
pub const alignment_bits = 3;
const Inner = packed struct {
gid: u32,
padding: u3 = 0,
groupname_len: u5,
pub fn groupnameLen(self: *const Inner) usize {
return @as(usize, self.groupname_len) + 1;
}
};
inner: *const Inner,
groupdata: []const u8,
members_offset: u64,
pub const Entry = struct {
group: PackedGroup,
next: ?[]const u8,
};
pub fn fromBytes(bytes: []const u8) error{Overflow}!Entry {
const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]);
const start_blob = @sizeOf(Inner);
const end_strings = @sizeOf(Inner) + inner.groupnameLen();
const members_offset = try compress.uvarint(bytes[end_strings..]);
const end_blob = end_strings + members_offset.bytes_read;
const next_start = pad.roundUp(usize, alignment_bits, end_blob);
var next: ?[]const u8 = null;
if (next_start < bytes.len)
next = bytes[next_start..];
return Entry{
.group = PackedGroup{
.inner = inner,
.groupdata = bytes[start_blob..end_strings],
.members_offset = members_offset.value,
},
.next = next,
};
}
fn validateUtf8(s: []const u8) InvalidRecord!void {
if (!std.unicode.utf8ValidateSlice(s))
return error.InvalidRecord;
}
pub const Iterator = struct {
section: ?[]const u8,
pub fn next(it: *Iterator) error{Overflow}!?PackedGroup {
if (it.section) |section| {
const entry = try fromBytes(section);
it.section = entry.next;
return entry.group;
}
return null;
}
};
pub fn iterator(section: []const u8) Iterator {
return Iterator{ .section = section };
}
pub fn gid(self: *const PackedGroup) u32 {
return self.inner.gid;
}
pub fn membersOffset(self: *const PackedGroup) u64 {
return self.members_offset;
}
pub fn name(self: *const PackedGroup) []const u8 {
return self.groupdata;
}
pub fn packTo(
arr: *ArrayList(u8),
group: GroupStored,
) error{ InvalidRecord, OutOfMemory }!void {
std.debug.assert(arr.items.len & 7 == 0);
try validate.utf8(group.name);
const len = try validate.downCast(u5, group.name.len - 1);
const inner = Inner{ .gid = group.gid, .groupname_len = len };
try arr.*.appendSlice(mem.asBytes(&inner));
try arr.*.appendSlice(group.name);
try compress.appendUvarint(arr, group.members_offset);
}
};
const testing = std.testing;
// someMembers constructs a bufset from an allocator and a list of strings.
pub fn someMembers(
allocator: Allocator,
members: []const []const u8,
) Allocator.Error!BufSet {
var bufset = BufSet.init(allocator);
errdefer bufset.deinit();
for (members) |member|
try bufset.insert(member);
return bufset;
}
test "PackedGroup alignment" {
try testing.expectEqual(@sizeOf(PackedGroup) * 8, @bitSizeOf(PackedGroup));
}
test "construct PackedGroups" {
var buf = ArrayList(u8).init(testing.allocator);
defer buf.deinit();
const groups = [_]GroupStored{
GroupStored{
.gid = 1000,
.name = "sudo",
.members_offset = 1,
},
GroupStored{
.gid = std.math.maxInt(u32),
.name = "Name" ** 8, // 32
.members_offset = std.math.maxInt(u64),
},
};
for (groups) |group| {
try PackedGroup.packTo(&buf, group);
try pad.arrayList(&buf, PackedGroup.alignment_bits);
}
var i: u29 = 0;
var it = PackedGroup.iterator(buf.items);
while (try it.next()) |group| : (i += 1) {
try testing.expectEqual(groups[i].gid, group.gid());
try testing.expectEqualStrings(groups[i].name, group.name());
try testing.expectEqual(groups[i].members_offset, group.membersOffset());
}
try testing.expectEqual(groups.len, i);
}
test "Group.clone" {
var allocator = testing.allocator;
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
var members = BufSet.init(allocator);
try members.insert("member1");
try members.insert("member2");
defer members.deinit();
var cloned = try members.cloneWithAllocator(arena.allocator());
cloned.remove("member1");
try cloned.insert("member4");
try testing.expect(members.contains("member1"));
try testing.expect(!members.contains("member4"));
try testing.expect(!cloned.contains("member1"));
try testing.expect(cloned.contains("member4"));
}

44
lib/header.zig
View File

@ -45,10 +45,10 @@ pub const Header = packed struct {
nblocks_groupmembers: u64,
nblocks_additional_gids: u64,
pub fn fromBytes(blob: []const u8) Invalid!Header {
pub fn fromBytes(blob: *const [@sizeOf(Header)]u8) Invalid!*const Header {
const self = mem.bytesAsValue(Header, blob);
if (!mem.eql(magic, blob[0..4]))
if (!mem.eql(u8, magic[0..4], blob[0..4]))
return error.InvalidMagic;
if (self.version != 0)
@ -75,27 +75,23 @@ test "bit header size is equal to @sizeOf(Header)" {
try testing.expectEqual(@sizeOf(Header) * 8, @bitSizeOf(Header));
}
test "header pack, unpack and validation" {
//const goodHeader = Header{};
test "header pack and unpack" {
const header1 = Header{
.nblocks_shell_blob = 0,
.num_shells = 0,
.num_groups = 0,
.num_users = 0,
.nblocks_bdz_gid = 0,
.nblocks_bdz_groupname = 0,
.nblocks_bdz_uid = 0,
.nblocks_bdz_username = 0,
.nblocks_groups = 0,
.nblocks_users = 0,
.nblocks_groupmembers = 0,
.nblocks_additional_gids = 1,
};
//const gotHeader = try Header.init(goodHeader.asArray());
//try testing.expectEqual(goodHeader, gotHeader);
//{
// var header = goodHeader;
// header.magic[0] = 0;
// try testing.expectError(error.InvalidMagic, Header.init(header.asArray()));
//}
//{
// var header = goodHeader;
// header.bom = 0x3412;
// try testing.expectError(error.InvalidBom, Header.init(header.asArray()));
//}
//{
// var header = goodHeader;
// header.offset_bdz_uid2user = 65;
// try testing.expectError(error.InvalidOffset, Header.init(header.asArray()));
//}
const bytes = mem.asBytes(&header1);
const header = try Header.fromBytes(bytes);
try testing.expectEqual(header1, header.*);
}

5
lib/test_all.zig
View File

@ -4,8 +4,9 @@ test "turbonss test suite" {
_ = @import("DB.zig");
_ = @import("Corpus.zig");
_ = @import("shell.zig");
_ = @import("user.zig");
_ = @import("group.zig");
_ = @import("User.zig");
_ = @import("PackedUser.zig");
_ = @import("Group.zig");
_ = @import("validate.zig");
_ = @import("padding.zig");
_ = @import("compress.zig");

353
lib/user.zig
View File

@ -1,353 +0,0 @@
const std = @import("std");
const pad = @import("padding.zig");
const validate = @import("validate.zig");
const compress = @import("compress.zig");
const shellImport = @import("shell.zig");
const InvalidRecord = validate.InvalidRecord;
const assert = std.debug.assert;
const mem = std.mem;
const math = std.math;
const Allocator = mem.Allocator;
const ArrayList = std.ArrayList;
const StringHashMap = std.StringHashMap;
// User is a convenient public struct for record construction and
// serialization.
pub const User = struct {
uid: u32,
gid: u32,
name: []const u8,
gecos: []const u8,
home: []const u8,
shell: []const u8,
// deep-clones a User record with a given Allocator.
pub fn clone(
self: *const User,
allocator: Allocator,
) Allocator.Error!User {
const stringdata = try allocator.alloc(u8, self.strlen());
const gecos_start = self.name.len;
const home_start = gecos_start + self.gecos.len;
const shell_start = home_start + self.home.len;
mem.copy(u8, stringdata[0..self.name.len], self.name);
mem.copy(u8, stringdata[gecos_start..], self.gecos);
mem.copy(u8, stringdata[home_start..], self.home);
mem.copy(u8, stringdata[shell_start..], self.shell);
return User{
.uid = self.uid,
.gid = self.gid,
.name = stringdata[0..self.name.len],
.gecos = stringdata[gecos_start .. gecos_start + self.gecos.len],
.home = stringdata[home_start .. home_start + self.home.len],
.shell = stringdata[shell_start .. shell_start + self.shell.len],
};
}
fn strlen(self: *const User) usize {
return self.name.len +
self.gecos.len +
self.home.len +
self.shell.len;
}
pub fn deinit(self: *User, allocator: Allocator) void {
const slice = self.home.ptr[0..self.strlen()];
allocator.free(slice);
self.* = undefined;
}
};
pub const PackedUser = struct {
const Self = @This();
pub const alignment_bits = 3;
const Inner = packed struct {
uid: u32,
gid: u32,
shell_len_or_idx: u8,
shell_here: bool,
name_is_a_suffix: bool,
home_len: u6,
name_len: u5,
gecos_len: u11,
fn homeLen(self: *const Inner) usize {
return @as(u32, self.home_len) + 1;
}
fn nameStart(self: *const Inner) usize {
const name_len = self.nameLen();
if (self.name_is_a_suffix) {
return self.homeLen() - name_len;
} else return self.homeLen();
}
fn nameLen(self: *const Inner) usize {
return @as(u32, self.name_len) + 1;
}
fn gecosStart(self: *const Inner) usize {
if (self.name_is_a_suffix) {
return self.homeLen();
} else return self.homeLen() + self.nameLen();
}
fn gecosLen(self: *const Inner) usize {
return self.gecos_len;
}
fn maybeShellStart(self: *const Inner) usize {
assert(self.shell_here);
return self.gecosStart() + self.gecosLen();
}
fn shellLen(self: *const Inner) usize {
return @as(u32, self.shell_len_or_idx) + 1;
}
// stringLength returns the length of the blob storing string values.
fn stringLength(self: *const Inner) usize {
var result: usize = self.homeLen() + self.gecosLen();
if (!self.name_is_a_suffix)
result += self.nameLen();
if (self.shell_here)
result += self.shellLen();
return result;
}
};
// PackedUser does not allocate; it re-interprets the "bytes" blob
// field. Both of those fields are pointers to "our representation" of
// that field.
inner: *const Inner,
bytes: []const u8,
additional_gids_offset: u64,
pub const Entry = struct {
user: Self,
next: ?[]const u8,
};
// TODO(motiejus) provide a way to return an entry without decoding the
// additional_gids_offset:
// - will not return the 'next' slice.
// - cannot throw an Overflow error.
pub fn fromBytes(bytes: []const u8) error{Overflow}!Entry {
const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]);
const start_blob = @sizeOf(Inner);
const end_strings = start_blob + inner.stringLength();
const gids_offset = try compress.uvarint(bytes[end_strings..]);
const end_blob = end_strings + gids_offset.bytes_read;
const nextStart = pad.roundUp(usize, alignment_bits, end_blob);
var next: ?[]const u8 = null;
if (nextStart < bytes.len)
next = bytes[nextStart..];
return Entry{
.user = Self{
.inner = inner,
.bytes = bytes[start_blob..end_blob],
.additional_gids_offset = gids_offset.value,
},
.next = next,
};
}
pub const Iterator = struct {
section: ?[]const u8,
shell_reader: shellImport.ShellReader,
pub fn next(it: *Iterator) error{Overflow}!?Self {
if (it.section) |section| {
const entry = try Self.fromBytes(section);
it.section = entry.next;
return entry.user;
}
return null;
}
};
pub fn iterator(section: []const u8, shell_reader: shellImport.ShellReader) Iterator {
return Iterator{ .section = section, .shell_reader = shell_reader };
}
// packTo packs the User record and copies it to the given byte slice.
// The slice must have at least maxRecordSize() bytes available. The
// slice is passed as a pointer, so it can be mutated.
pub fn packTo(
arr: *ArrayList(u8),
user: User,
additional_gids_offset: u64,
idxFn: StringHashMap(u8),
) error{ InvalidRecord, OutOfMemory }!void {
std.debug.assert(arr.items.len & 7 == 0);
// function arguments are consts. We need to mutate the underlying
// slice, so passing it via pointer instead.
const home_len = try validate.downCast(u6, user.home.len - 1);
const name_len = try validate.downCast(u5, user.name.len - 1);
const shell_len = try validate.downCast(u8, user.shell.len - 1);
const gecos_len = try validate.downCast(u8, user.gecos.len);
try validate.utf8(user.home);
try validate.utf8(user.name);
try validate.utf8(user.shell);
try validate.utf8(user.gecos);
const inner = Inner{
.uid = user.uid,
.gid = user.gid,
.shell_here = idxFn.get(user.shell) == null,
.shell_len_or_idx = idxFn.get(user.shell) orelse shell_len,
.home_len = home_len,
.name_is_a_suffix = mem.endsWith(u8, user.home, user.name),
.name_len = name_len,
.gecos_len = gecos_len,
};
const innerBytes = mem.asBytes(&inner);
try arr.*.appendSlice(innerBytes[0..@sizeOf(Inner)]);
try arr.*.appendSlice(user.home);
if (!inner.name_is_a_suffix)
try arr.*.appendSlice(user.name);
try arr.*.appendSlice(user.gecos);
if (inner.shell_here)
try arr.*.appendSlice(user.shell);
try compress.appendUvarint(arr, additional_gids_offset);
}
pub fn uid(self: Self) u32 {
return self.inner.uid;
}
pub fn gid(self: Self) u32 {
return self.inner.gid;
}
pub fn additionalGidsOffset(self: Self) u64 {
return self.additional_gids_offset;
}
pub fn home(self: Self) []const u8 {
return self.bytes[0..self.inner.homeLen()];
}
pub fn name(self: Self) []const u8 {
const name_pos = self.inner.nameStart();
const name_len = self.inner.nameLen();
return self.bytes[name_pos .. name_pos + name_len];
}
pub fn gecos(self: Self) []const u8 {
const gecos_pos = self.inner.gecosStart();
const gecos_len = self.inner.gecosLen();
return self.bytes[gecos_pos .. gecos_pos + gecos_len];
}
pub fn shell(self: Self, shell_reader: shellImport.ShellReader) []const u8 {
if (self.inner.shell_here) {
const shell_pos = self.inner.maybeShellStart();
const shell_len = self.inner.shellLen();
return self.bytes[shell_pos .. shell_pos + shell_len];
}
return shell_reader.get(self.inner.shell_len_or_idx);
}
};
const testing = std.testing;
test "PackedUser internal and external alignment" {
try testing.expectEqual(
@sizeOf(PackedUser.Inner) * 8,
@bitSizeOf(PackedUser.Inner),
);
}
fn testShellIndex(allocator: Allocator) StringHashMap(u8) {
var result = StringHashMap(u8).init(allocator);
result.put("/bin/bash", 0) catch unreachable;
result.put("/bin/zsh", 1) catch unreachable;
return result;
}
const test_shell_reader = shellImport.ShellReader{
.blob = "/bin/bash/bin/zsh",
.index = &[_]u16{ 0, 9, 17 },
};
test "construct PackedUser section" {
var buf = ArrayList(u8).init(testing.allocator);
defer buf.deinit();
const users = [_]User{ User{
.uid = 1000,
.gid = 1000,
.name = "vidmantas",
.gecos = "Vidmantas Kaminskas",
.home = "/home/vidmantas",
.shell = "/bin/bash",
}, User{
.uid = 1001,
.gid = 1001,
.name = "svc-foo",
.gecos = "Service Account",
.home = "/home/service1",
.shell = "/usr/bin/nologin",
}, User{
.uid = 0,
.gid = math.maxInt(u32),
.name = "Name" ** 8,
.gecos = "Gecos" ** 51,
.home = "Home" ** 16,
.shell = "She.LllL" ** 32,
}, User{
.uid = 1002,
.gid = 1002,
.name = "svc-bar",
.gecos = "",
.home = "/",
.shell = "/bin/zsh",
} };
var shellIndex = testShellIndex(testing.allocator);
const additional_gids = math.maxInt(u64);
defer shellIndex.deinit();
for (users) |user| {
try PackedUser.packTo(&buf, user, additional_gids, shellIndex);
try pad.arrayList(&buf, PackedUser.alignment_bits);
}
var i: u29 = 0;
var it1 = PackedUser.iterator(buf.items, test_shell_reader);
while (try it1.next()) |user| : (i += 1) {
try testing.expectEqual(users[i].uid, user.uid());
try testing.expectEqual(users[i].gid, user.gid());
try testing.expectEqual(user.additionalGidsOffset(), additional_gids);
try testing.expectEqualStrings(users[i].name, user.name());
try testing.expectEqualStrings(users[i].gecos, user.gecos());
try testing.expectEqualStrings(users[i].home, user.home());
try testing.expectEqualStrings(users[i].shell, user.shell(test_shell_reader));
}
try testing.expectEqual(users.len, i);
}
test "User.clone" {
var allocator = testing.allocator;
const user = User{
.uid = 1000,
.gid = 1000,
.name = "vidmantas",
.gecos = "Vidmantas Kaminskas",
.home = "/home/vidmantas",
.shell = "/bin/bash",
};
var user2 = try user.clone(allocator);
defer user2.deinit(allocator);
try testing.expectEqualStrings(user.shell, "/bin/bash");
}