This commit is contained in:
Motiejus Jakštys 2023-02-02 07:04:21 -08:00
parent 5d3dfdc8dc
commit 0ecd6172fc
16 changed files with 227 additions and 197 deletions

View File

@ -4,14 +4,15 @@ const zbs = std.build;
pub fn build(b: *zbs.Builder) void {
const target = b.standardTargetOptions(.{});
const mode = b.standardReleaseOptions();
b.use_stage1 = true;
const optimize = b.standardOptimizeOption(.{});
const strip = b.option(bool, "strip", "Omit debug information") orelse false;
const cmph = b.addStaticLibrary("cmph", null);
cmph.setTarget(target);
cmph.setBuildMode(mode);
const cmph = b.addStaticLibrary(.{
.name = "cmph",
.target = target,
.optimize = optimize,
});
cmph.linkLibC();
cmph.addCSourceFiles(&.{
"deps/cmph/src/bdz.c",
@ -49,9 +50,11 @@ pub fn build(b: *zbs.Builder) void {
cmph.addIncludePath("deps/cmph/src");
cmph.addIncludePath("include/deps/cmph");
const bdz = b.addStaticLibrary("bdz", null);
bdz.setTarget(target);
bdz.setBuildMode(mode);
const bdz = b.addStaticLibrary(.{
.name = "bdz",
.target = target,
.optimize = optimize,
});
bdz.linkLibC();
bdz.addCSourceFiles(&.{
"deps/bdz_read.c",
@ -70,56 +73,72 @@ pub fn build(b: *zbs.Builder) void {
bdz.want_lto = true;
{
const exe = b.addExecutable("turbonss-unix2db", "src/turbonss-unix2db.zig");
const exe = b.addExecutable(.{
.name = "turbonss-unix2db",
.root_source_file = .{ .path = "src/turbonss-unix2db.zig" },
.target = target,
.optimize = optimize,
});
exe.compress_debug_sections = .zlib;
exe.strip = strip;
exe.want_lto = true;
exe.setTarget(target);
exe.setBuildMode(mode);
addCmphDeps(exe, cmph);
exe.install();
}
{
const exe = b.addExecutable("turbonss-analyze", "src/turbonss-analyze.zig");
const exe = b.addExecutable(.{
.name = "turbonss-analyze",
.root_source_file = .{ .path = "src/turbonss-analyze.zig" },
.target = target,
.optimize = optimize,
});
exe.compress_debug_sections = .zlib;
exe.strip = strip;
exe.want_lto = true;
exe.setTarget(target);
exe.setBuildMode(mode);
exe.install();
}
{
const exe = b.addExecutable("turbonss-makecorpus", "src/turbonss-makecorpus.zig");
const exe = b.addExecutable(.{
.name = "turbonss-makecorpus",
.root_source_file = .{ .path = "src/turbonss-makecorpus.zig" },
.target = target,
.optimize = optimize,
});
exe.compress_debug_sections = .zlib;
exe.strip = strip;
exe.want_lto = true;
exe.setTarget(target);
exe.setBuildMode(mode);
exe.install();
}
{
const exe = b.addExecutable("turbonss-getent", "src/turbonss-getent.zig");
const exe = b.addExecutable(.{
.name = "turbonss-getent",
.root_source_file = .{ .path = "src/turbonss-getent.zig" },
.target = target,
.optimize = optimize,
});
exe.compress_debug_sections = .zlib;
exe.strip = strip;
exe.want_lto = true;
exe.linkLibC();
exe.linkLibrary(bdz);
exe.addIncludePath("deps/cmph/src");
exe.setTarget(target);
exe.setBuildMode(mode);
exe.install();
}
{
const so = b.addSharedLibrary("nss_turbo", "src/libnss.zig", .{
.versioned = builtin.Version{
const so = b.addSharedLibrary(.{
.name = "nss_turbo",
.root_source_file = .{ .path = "src/libnss.zig" },
.version = builtin.Version{
.major = 2,
.minor = 0,
.patch = 0,
},
.target = target,
.optimize = optimize,
});
so.compress_debug_sections = .zlib;
so.strip = strip;
@ -127,13 +146,15 @@ pub fn build(b: *zbs.Builder) void {
so.linkLibC();
so.linkLibrary(bdz);
so.addIncludePath("deps/cmph/src");
so.setTarget(target);
so.setBuildMode(mode);
so.install();
}
{
const src_test = b.addTest("src/test_all.zig");
const src_test = b.addTest(.{
.root_source_file = .{ .path = "src/test_all.zig" },
.target = target,
.optimize = optimize,
});
addCmphDeps(src_test, cmph);
const test_step = b.step("test", "Run the tests");
test_step.dependOn(&src_test.step);

View File

@ -199,7 +199,7 @@ pub fn init(
user2groups_final.len = users.len;
for (user2groups) |*usergroups, i| {
sort.sort(u32, usergroups.items, {}, comptime sort.asc(u32));
user2groups_final[i] = usergroups.toOwnedSlice(allocator);
user2groups_final[i] = try usergroups.toOwnedSlice(allocator);
}
return Corpus{

View File

@ -6,7 +6,7 @@ const meta = std.meta;
const sort = std.sort;
const assert = std.debug.assert;
const Allocator = std.mem.Allocator;
const ArrayList = std.ArrayList;
const ArrayListAligned = std.ArrayListAligned;
const AutoHashMap = std.AutoHashMap;
const BoundedArray = std.BoundedArray;
@ -37,20 +37,20 @@ const zeroes = &[_]u8{0} ** section_length;
const DB = @This();
// All sections, as they end up in the DB. Order is important.
header: *const Header,
bdz_gid: []const u8,
bdz_groupname: []const u8,
bdz_uid: []const u8,
bdz_username: []const u8,
idx_gid2group: []const u32,
idx_groupname2group: []const u32,
idx_uid2user: []const u32,
idx_name2user: []const u32,
shell_index: []const u16,
shell_blob: []const u8,
groups: []const u8,
users: []const u8,
groupmembers: []const u8,
additional_gids: []const u8,
bdz_gid: []align(8) const u8,
bdz_groupname: []align(8) const u8,
bdz_uid: []align(8) const u8,
bdz_username: []align(8) const u8,
idx_gid2group: []align(8) const u32,
idx_groupname2group: []align(8) const u32,
idx_uid2user: []align(8) const u32,
idx_name2user: []align(8) const u32,
shell_index: []align(8) const u16,
shell_blob: []align(8) const u8,
groups: []align(8) const u8,
users: []align(8) const u8,
groupmembers: []align(8) const u8,
additional_gids: []align(8) const u8,
pub fn fromCorpus(
allocator: Allocator,
@ -79,10 +79,12 @@ pub fn fromCorpus(
var shell = try shellSections(allocator, corpus);
defer shell.deinit();
const shell_index = try allocator.dupe(u16, shell.index.constSlice());
const shell_index = try allocator.alignedAlloc(u16, 8, shell.index.len);
mem.copy(u16, shell_index, shell.index.constSlice());
errdefer allocator.free(shell_index);
const shell_blob = try allocator.dupe(u8, shell.blob.constSlice());
const shell_blob = try allocator.alignedAlloc(u8, 8, shell.blob.len);
mem.copy(u8, shell_blob, shell.blob.constSlice());
errdefer allocator.free(shell_blob);
const additional_gids = try additionalGids(allocator, corpus);
@ -180,12 +182,12 @@ pub fn deinit(self: *DB, allocator: Allocator) void {
}
const DB_fields = meta.fields(DB);
pub fn iov(self: *const DB) BoundedArray(os.iovec_const, DB_fields.len * 2) {
pub fn iov(self: *align(8) const DB) BoundedArray(os.iovec_const, DB_fields.len * 2) {
var result = BoundedArray(os.iovec_const, DB_fields.len * 2).init(0) catch unreachable;
inline for (DB_fields) |field| {
comptime assertDefinedLayout(field.field_type);
comptime assertDefinedLayout(field.type);
const value = @field(self, field.name);
const bytes: []const u8 = switch (@TypeOf(value)) {
const bytes: []align(8) const u8 = switch (@TypeOf(value)) {
*const Header => mem.asBytes(value),
else => mem.sliceAsBytes(value),
};
@ -248,12 +250,11 @@ pub fn fieldOffsets(lengths: DBNumbers) DBNumbers {
var result: DBNumbers = undefined;
result.header = 0;
var offset = comptime nblocks_n(u64, @sizeOf(Header));
inline for (DB_fields[0..]) |field, i| {
comptime {
assert(mem.eql(u8, field.name, meta.fields(DBNumbers)[i].name));
if (mem.eql(u8, field.name, "header")) continue;
}
// skipping header (so index 1). This used to be an inline for with stage1,
// but that and a comptime assertion crashes the compiler as of
// 0.11.0-dev.1580+a5b34a61a
inline for (DB_fields[1..]) |field, i| {
assert(mem.eql(u8, field.name, meta.fields(DBNumbers)[i + 1].name));
@field(result, field.name) = offset;
offset += @field(lengths, field.name);
}
@ -271,8 +272,8 @@ pub fn fromBytes(buf: []align(8) const u8) InvalidHeader!DB {
const start_block = @field(offsets, field.name);
const end = (start_block + @field(lengths, field.name)) << section_length_bits;
const start = start_block << section_length_bits;
const slice_type = meta.Child(field.field_type);
const value = mem.bytesAsSlice(slice_type, buf[start..end]);
const slice_type = meta.Child(field.type);
const value = mem.bytesAsSlice(slice_type, @alignCast(8, buf[start..end]));
@field(result, field.name) = value;
}
@ -343,7 +344,7 @@ pub fn packCGroup(self: *const DB, group: *const PackedGroup, buf: []u8) error{B
var i: usize = 0;
while (it.nextMust()) |member_offset| : (i += 1) {
const entry = PackedUser.fromBytes(self.users[member_offset << 3 ..]);
const entry = PackedUser.fromBytes(@alignCast(8, self.users[member_offset << 3 ..]));
const start = buf_offset;
const name = entry.user.name();
if (buf_offset + name.len + 1 > buf.len)
@ -374,8 +375,7 @@ pub fn getGroupByName(self: *const DB, name: []const u8) ?PackedGroup {
const idx = bdz.search(self.bdz_groupname, name);
if (idx >= self.header.num_groups) return null;
const offset = self.idx_groupname2group[idx];
const nbits = PackedGroup.alignment_bits;
const group = PackedGroup.fromBytes(self.groups[offset << nbits ..]).group;
const group = PackedGroup.fromBytes(@alignCast(8, self.groups[offset << 3 ..])).group;
if (!mem.eql(u8, name, group.name())) return null;
return group;
}
@ -384,8 +384,7 @@ pub fn getGroupByGid(self: *const DB, gid: u32) ?PackedGroup {
const idx = bdz.search_u32(self.bdz_gid, gid);
if (idx >= self.header.num_groups) return null;
const offset = self.idx_gid2group[idx];
const nbits = PackedGroup.alignment_bits;
const group = PackedGroup.fromBytes(self.groups[offset << nbits ..]).group;
const group = PackedGroup.fromBytes(@alignCast(8, self.groups[offset << 3 ..])).group;
if (gid != group.gid()) return null;
return group;
}
@ -467,8 +466,7 @@ pub fn getUserByName(self: *const DB, name: []const u8) ?PackedUser {
// bdz may return a hash that's bigger than the number of users
if (idx >= self.header.num_users) return null;
const offset = self.idx_name2user[idx];
const nbits = PackedUser.alignment_bits;
const user = PackedUser.fromBytes(self.users[offset << nbits ..]).user;
const user = PackedUser.fromBytes(@alignCast(8, self.users[offset << 3 ..])).user;
if (!mem.eql(u8, name, user.name())) return null;
return user;
}
@ -483,8 +481,7 @@ pub fn getUserByUid(self: *const DB, uid: u32) ?PackedUser {
const idx = bdz.search_u32(self.bdz_uid, uid);
if (idx >= self.header.num_users) return null;
const offset = self.idx_uid2user[idx];
const nbits = PackedUser.alignment_bits;
const user = PackedUser.fromBytes(self.users[offset << nbits ..]).user;
const user = PackedUser.fromBytes(@alignCast(8, self.users[offset << 3 ..])).user;
if (uid != user.uid()) return null;
return user;
}
@ -508,20 +505,20 @@ fn shellSections(
const AdditionalGids = struct {
// user index -> offset in blob
idx2offset: []const u64,
idx2offset: []align(8) const u64,
// compressed user gids blob. A blob contains N <= users.len items,
// an item is:
// len: varint
// gid: [varint]varint,
// ... and the gid list is delta-compressed.
blob: []const u8,
blob: []align(8) const u8,
};
fn additionalGids(
allocator: Allocator,
corpus: *const Corpus,
) error{OutOfMemory}!AdditionalGids {
var blob = ArrayList(u8).init(allocator);
var blob = ArrayListAligned(u8, 8).init(allocator);
errdefer blob.deinit();
var idx2offset = try allocator.alloc(u64, corpus.users.len);
errdefer allocator.free(idx2offset);
@ -529,7 +526,7 @@ fn additionalGids(
// zero'th entry is empty, so groupless users can refer to it.
try compress.appendUvarint(&blob, 0);
var scratch = try allocator.alloc(u32, 256);
var scratch = try allocator.alignedAlloc(u32, 8, 256);
var scratch_allocated: bool = true;
defer if (scratch_allocated) allocator.free(scratch);
for (corpus.user2groups) |usergroups, user_idx| {
@ -541,7 +538,7 @@ fn additionalGids(
if (scratch.len < usergroups.len) {
allocator.free(scratch);
scratch_allocated = false;
scratch = try allocator.alloc(u32, usergroups.len);
scratch = try allocator.alignedAlloc(u32, 8, usergroups.len);
scratch_allocated = true;
}
scratch.len = usergroups.len;
@ -558,7 +555,7 @@ fn additionalGids(
return AdditionalGids{
.idx2offset = idx2offset,
.blob = blob.toOwnedSlice(),
.blob = try blob.toOwnedSlice(),
};
}
@ -566,8 +563,8 @@ const UsersSection = struct {
// number of users in this section
len: u32,
// user index -> offset in blob
idx2offset: []const u32,
blob: []const u8,
idx2offset: []align(8) const u32,
blob: []align(8) const u8,
};
fn usersSection(
@ -576,15 +573,15 @@ fn usersSection(
gids: *const AdditionalGids,
shells: *const ShellSections,
) error{ OutOfMemory, InvalidRecord, TooMany }!UsersSection {
var idx2offset = try allocator.alloc(u32, corpus.users.len);
var idx2offset = try allocator.alignedAlloc(u32, 8, corpus.users.len);
errdefer allocator.free(idx2offset);
// as of writing each user takes 12 bytes + blobs + padding, padded to
// 8 bytes. 24 is an optimistic lower bound for an average record size.
var blob = try ArrayList(u8).initCapacity(allocator, 24 * corpus.users.len);
var blob = try ArrayListAligned(u8, 8).initCapacity(allocator, 24 * corpus.users.len);
errdefer blob.deinit();
var i: usize = 0;
while (i < corpus.users.len) : (i += 1) {
// TODO: this is inefficient by calling `.slice()` on every iteration
// TODO: this may be inefficient by calling `.slice()` on every iteration?
const user = corpus.users.get(i);
const user_offset = math.cast(u35, blob.items.len) orelse return error.TooMany;
assert(user_offset & 7 == 0);
@ -600,7 +597,7 @@ fn usersSection(
return UsersSection{
.len = @intCast(u32, corpus.users.len),
.idx2offset = idx2offset,
.blob = blob.toOwnedSlice(),
.blob = try blob.toOwnedSlice(),
};
}
@ -608,7 +605,7 @@ const GroupMembers = struct {
// group index to it's offset in blob
idx2offset: []const u64,
// members are delta-varint encoded byte-offsets to the user struct
blob: []const u8,
blob: []align(8) const u8,
};
fn groupMembers(
@ -618,12 +615,12 @@ fn groupMembers(
) error{OutOfMemory}!GroupMembers {
var idx2offset = try allocator.alloc(u64, corpus.groups.len);
errdefer allocator.free(idx2offset);
var blob = ArrayList(u8).init(allocator);
var blob = ArrayListAligned(u8, 8).init(allocator);
errdefer blob.deinit();
// zero'th entry is empty, so empty groups can refer to it
try compress.appendUvarint(&blob, 0);
var scratch = try ArrayList(u32).initCapacity(allocator, 1024);
var scratch = try ArrayListAligned(u32, 8).initCapacity(allocator, 1024);
defer scratch.deinit();
for (corpus.group2users) |members, group_idx| {
@ -647,7 +644,7 @@ fn groupMembers(
}
return GroupMembers{
.idx2offset = idx2offset,
.blob = blob.toOwnedSlice(),
.blob = try blob.toOwnedSlice(),
};
}
@ -655,8 +652,8 @@ const GroupsSection = struct {
// number of groups in this section
len: u32,
// group index -> offset in blob
idx2offset: []const u32,
blob: []const u8,
idx2offset: []align(8) const u32,
blob: []align(8) const u8,
};
fn groupsSection(
@ -664,10 +661,10 @@ fn groupsSection(
corpus: *const Corpus,
members_offset: []const u64,
) error{ OutOfMemory, InvalidRecord }!GroupsSection {
var idx2offset = try allocator.alloc(u32, corpus.groups.len);
var idx2offset = try allocator.alignedAlloc(u32, 8, corpus.groups.len);
errdefer allocator.free(idx2offset);
var blob = try ArrayList(u8).initCapacity(allocator, 8 * corpus.groups.len);
var blob = try ArrayListAligned(u8, 8).initCapacity(allocator, 8 * corpus.groups.len);
errdefer blob.deinit();
var i: usize = 0;
@ -689,7 +686,7 @@ fn groupsSection(
return GroupsSection{
.len = @intCast(u32, corpus.groups.len),
.idx2offset = idx2offset,
.blob = blob.toOwnedSlice(),
.blob = try blob.toOwnedSlice(),
};
}
@ -707,14 +704,14 @@ fn bdzIdx(
packed_mphf: []const u8,
keys: []const T,
idx2offset: []const u32,
) error{OutOfMemory}![]const u32 {
) error{OutOfMemory}![]align(8) const u32 {
const search_fn = switch (T) {
u32 => bdz.search_u32,
[]const u8 => bdz.search,
else => unreachable,
};
assert(keys.len <= math.maxInt(u32));
var result = try allocator.alloc(u32, keys.len);
var result = try allocator.alignedAlloc(u32, 8, keys.len);
errdefer allocator.free(result);
for (keys) |key, i|
result[search_fn(packed_mphf, key)] = idx2offset[i];
@ -751,7 +748,7 @@ fn assertDefinedLayout(comptime T: type) void {
if (meta.containerLayout(T) == .Auto)
@compileError("layout of " ++ @typeName(T) ++ " is undefined");
for (meta.fields(T)) |field|
assertDefinedLayout(field.field_type);
assertDefinedLayout(field.type);
},
else => @compileError("unexpected type " ++ @typeName(T)),
},
@ -793,7 +790,7 @@ test "DB getgrnam/getgrgid" {
var errc = ErrCtx{};
var db = try DB.fromCorpus(testing.allocator, &corpus, &errc);
defer db.deinit(testing.allocator);
var buf = try testing.allocator.alloc(u8, db.getgrBufsize());
var buf = try testing.allocator.alignedAlloc(u8, 8, db.getgrBufsize());
defer testing.allocator.free(buf);
{
@ -835,7 +832,7 @@ test "DB getpwnam/getpwuid" {
var errc = ErrCtx{};
var db = try DB.fromCorpus(testing.allocator, &corpus, &errc);
defer db.deinit(testing.allocator);
var buf = try testing.allocator.alloc(u8, db.getpwBufsize());
var buf = try testing.allocator.alignedAlloc(u8, 8, db.getpwBufsize());
defer testing.allocator.free(buf);
{

View File

@ -135,7 +135,7 @@ pub const CGroup = extern struct {
};
// size of the pointer to a single member.
pub const ptr_size = @sizeOf(meta.Child(meta.fieldInfo(CGroup, .gr_mem).field_type));
pub const ptr_size = @sizeOf(meta.Child(meta.fieldInfo(CGroup, .gr_mem).type));
const testing = std.testing;

View File

@ -3,7 +3,7 @@ const std = @import("std");
const mem = std.mem;
const assert = std.debug.assert;
const Allocator = mem.Allocator;
const ArrayList = std.ArrayList;
const ArrayListAligned = std.ArrayListAligned;
const BufSet = std.BufSet;
const pad = @import("padding.zig");
@ -40,7 +40,7 @@ pub const Entry = struct {
end: usize,
};
pub fn fromBytes(bytes: []const u8) Entry {
pub fn fromBytes(bytes: []align(8) const u8) Entry {
const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]);
const start_blob = @sizeOf(Inner);
const end_strings = @sizeOf(Inner) + inner.groupnameLen();
@ -65,7 +65,7 @@ fn validateUtf8(s: []const u8) InvalidRecord!void {
}
pub const Iterator = struct {
section: []const u8,
section: []align(8) const u8,
next_start: usize = 0,
idx: u32 = 0,
total: u32,
@ -73,7 +73,7 @@ pub const Iterator = struct {
pub fn next(it: *Iterator) ?PackedGroup {
if (it.idx == it.total) return null;
const entry = fromBytes(it.section[it.next_start..]);
const entry = fromBytes(@alignCast(8, it.section[it.next_start..]));
it.idx += 1;
it.next_start += entry.end;
it.advanced_by = entry.end;
@ -88,7 +88,7 @@ pub const Iterator = struct {
}
};
pub fn iterator(section: []const u8, total: u32) Iterator {
pub fn iterator(section: []align(8) const u8, total: u32) Iterator {
return Iterator{
.section = section,
.total = total,
@ -108,7 +108,7 @@ pub inline fn name(self: *const PackedGroup) []const u8 {
}
pub fn packTo(
arr: *ArrayList(u8),
arr: *ArrayListAligned(u8, 8),
group: GroupStored,
) error{ InvalidRecord, OutOfMemory }!void {
std.debug.assert(arr.items.len & 7 == 0);
@ -127,7 +127,7 @@ test "PackedGroup alignment" {
}
test "PackedGroup construct" {
var buf = ArrayList(u8).init(testing.allocator);
var buf = ArrayListAligned(u8, 8).init(testing.allocator);
defer buf.deinit();
const groups = [_]GroupStored{

View File

@ -3,7 +3,7 @@ const assert = std.debug.assert;
const mem = std.mem;
const math = std.math;
const Allocator = mem.Allocator;
const ArrayList = std.ArrayList;
const ArrayListAligned = std.ArrayListAligned;
const StringHashMap = std.StringHashMap;
const fieldInfo = std.meta.fieldInfo;
@ -72,11 +72,11 @@ const Inner = packed struct {
}
};
// PackedUser does not allocate; it re-interprets the "bytes" blob
// PackedUser does not allocate; it re-interprets the var_payload
// field. Both of those fields are pointers to "our representation" of
// that field.
inner: *const Inner,
bytes: []const u8,
inner: *align(8) const Inner,
var_payload: []const u8,
additional_gids_offset: u64,
pub const Entry = struct {
@ -84,27 +84,27 @@ pub const Entry = struct {
end: usize,
};
pub fn fromBytes(bytes: []const u8) Entry {
const inner = mem.bytesAsValue(Inner, bytes[0..@sizeOf(Inner)]);
const start_blob = @sizeOf(Inner);
const end_strings = start_blob + inner.stringLength();
const gids_offset = compress.uvarint(bytes[end_strings..]) catch |err| switch (err) {
pub fn fromBytes(blob: []align(8) const u8) Entry {
const start_var_payload = @bitSizeOf(Inner) / 8;
const inner = @ptrCast(*align(8) const Inner, blob[0..start_var_payload]);
const end_strings = start_var_payload + inner.stringLength();
const gids_offset = compress.uvarint(blob[end_strings..]) catch |err| switch (err) {
error.Overflow => unreachable,
};
const end_blob = end_strings + gids_offset.bytes_read;
const end_payload = end_strings + gids_offset.bytes_read;
return Entry{
.user = PackedUser{
.inner = inner,
.bytes = bytes[start_blob..end_blob],
.var_payload = blob[start_var_payload..end_payload],
.additional_gids_offset = gids_offset.value,
},
.end = pad.roundUp(usize, alignment_bits, end_blob),
.end = pad.roundUp(usize, alignment_bits, end_payload),
};
}
pub const Iterator = struct {
section: []const u8,
section: []align(8) const u8,
next_start: usize = 0,
shell_reader: ShellReader,
idx: u32 = 0,
@ -113,7 +113,7 @@ pub const Iterator = struct {
pub fn next(it: *Iterator) ?PackedUser {
if (it.idx == it.total) return null;
const entry = fromBytes(it.section[it.next_start..]);
const entry = fromBytes(@alignCast(8, it.section[it.next_start..]));
it.idx += 1;
it.next_start += entry.end;
it.advanced_by = entry.end;
@ -128,7 +128,7 @@ pub const Iterator = struct {
}
};
pub fn iterator(section: []const u8, total: u32, shell_reader: ShellReader) Iterator {
pub fn iterator(section: []align(8) const u8, total: u32, shell_reader: ShellReader) Iterator {
return Iterator{
.section = section,
.total = total,
@ -138,7 +138,7 @@ pub fn iterator(section: []const u8, total: u32, shell_reader: ShellReader) Iter
// packTo packs the User record and copies it to the given arraylist.
pub fn packTo(
arr: *ArrayList(u8),
arr: *ArrayListAligned(u8, 8),
user: User,
additional_gids_offset: u64,
idxFn: StringHashMap(u8),
@ -146,10 +146,10 @@ pub fn packTo(
std.debug.assert(arr.items.len & 7 == 0);
// function arguments are consts. We need to mutate the underlying
// slice, so passing it via pointer instead.
const home_len = try validate.downCast(fieldInfo(Inner, .home_len).field_type, user.home.len - 1);
const name_len = try validate.downCast(fieldInfo(Inner, .name_len).field_type, user.name.len - 1);
const shell_len = try validate.downCast(fieldInfo(Inner, .shell_len_or_idx).field_type, user.shell.len - 1);
const gecos_len = try validate.downCast(fieldInfo(Inner, .gecos_len).field_type, user.gecos.len);
const home_len = try validate.downCast(fieldInfo(Inner, .home_len).type, user.home.len - 1);
const name_len = try validate.downCast(fieldInfo(Inner, .name_len).type, user.name.len - 1);
const shell_len = try validate.downCast(fieldInfo(Inner, .shell_len_or_idx).type, user.shell.len - 1);
const gecos_len = try validate.downCast(fieldInfo(Inner, .gecos_len).type, user.gecos.len);
try validate.utf8(user.home);
try validate.utf8(user.name);
@ -166,7 +166,7 @@ pub fn packTo(
.name_len = name_len,
.gecos_len = gecos_len,
};
try arr.*.appendSlice(mem.asBytes(&inner)[0..@sizeOf(Inner)]);
try arr.*.appendSlice(mem.asBytes(&inner)[0 .. @bitSizeOf(Inner) / 8]);
try arr.*.appendSlice(user.home);
if (!inner.name_is_a_suffix)
@ -190,26 +190,26 @@ pub fn additionalGidsOffset(self: PackedUser) u64 {
}
pub fn home(self: PackedUser) []const u8 {
return self.bytes[0..self.inner.homeLen()];
return self.var_payload[0..self.inner.homeLen()];
}
pub fn name(self: PackedUser) []const u8 {
const name_pos = self.inner.nameStart();
const name_len = self.inner.nameLen();
return self.bytes[name_pos .. name_pos + name_len];
return self.var_payload[name_pos .. name_pos + name_len];
}
pub fn gecos(self: PackedUser) []const u8 {
const gecos_pos = self.inner.gecosStart();
const gecos_len = self.inner.gecosLen();
return self.bytes[gecos_pos .. gecos_pos + gecos_len];
return self.var_payload[gecos_pos .. gecos_pos + gecos_len];
}
pub fn shell(self: PackedUser, shell_reader: ShellReader) []const u8 {
if (self.inner.shell_here) {
const shell_pos = self.inner.maybeShellStart();
const shell_len = self.inner.shellLen();
return self.bytes[shell_pos .. shell_pos + shell_len];
return self.var_payload[shell_pos .. shell_pos + shell_len];
}
return shell_reader.get(self.inner.shell_len_or_idx);
}
@ -226,25 +226,18 @@ pub fn toUser(self: *const PackedUser, shell_reader: ShellReader) User {
};
}
pub const max_home_len = math.maxInt(fieldInfo(Inner, .home_len).field_type) + 1;
pub const max_name_len = math.maxInt(fieldInfo(Inner, .name_len).field_type) + 1;
pub const max_gecos_len = math.maxInt(fieldInfo(Inner, .gecos_len).field_type);
pub const max_home_len = math.maxInt(fieldInfo(Inner, .home_len).type) + 1;
pub const max_name_len = math.maxInt(fieldInfo(Inner, .name_len).type) + 1;
pub const max_gecos_len = math.maxInt(fieldInfo(Inner, .gecos_len).type);
pub const max_str_len =
math.maxInt(fieldInfo(Inner, .shell_len_or_idx).field_type) + 1 +
math.maxInt(fieldInfo(Inner, .shell_len_or_idx).type) + 1 +
max_home_len +
max_name_len +
max_gecos_len;
const testing = std.testing;
test "PackedUser internal and external alignment" {
try testing.expectEqual(
@sizeOf(PackedUser.Inner) * 8,
@bitSizeOf(PackedUser.Inner),
);
}
fn testShellIndex(allocator: Allocator) StringHashMap(u8) {
var result = StringHashMap(u8).init(allocator);
result.put("/bin/bash", 0) catch unreachable;
@ -252,13 +245,14 @@ fn testShellIndex(allocator: Allocator) StringHashMap(u8) {
return result;
}
const test_shell_reader_index: [3]u16 align(8) = .{ 0, 9, 17 };
const test_shell_reader = ShellReader{
.blob = "/bin/bash/bin/zsh",
.index = &[_]u16{ 0, 9, 17 },
.index = &test_shell_reader_index,
};
test "PackedUser pack max_user" {
var arr = ArrayList(u8).init(testing.allocator);
var arr = ArrayListAligned(u8, 8).init(testing.allocator);
defer arr.deinit();
var idx_noop = StringHashMap(u8).init(testing.allocator);
@ -268,7 +262,7 @@ test "PackedUser pack max_user" {
}
test "PackedUser construct section" {
var buf = ArrayList(u8).init(testing.allocator);
var buf = ArrayListAligned(u8, 8).init(testing.allocator);
defer buf.deinit();
const users = [_]User{ User{

View File

@ -24,7 +24,7 @@ extern fn cmph_destroy(mphf: [*]u8) void;
// pack packs cmph hashes for the given input and returns a slice ("cmph pack
// minus first 4 bytes") for further storage. The slice must be freed by the
// caller.
pub fn pack(allocator: Allocator, input: [][*:0]const u8) error{OutOfMemory}![]const u8 {
pub fn pack(allocator: Allocator, input: [][*:0]const u8) error{OutOfMemory}![]align(8) const u8 {
const input_len = @intCast(c_uint, input.len);
var source = cmph_io_vector_adapter(input.ptr, input_len);
defer cmph_io_vector_adapter_destroy(source);
@ -35,7 +35,7 @@ pub fn pack(allocator: Allocator, input: [][*:0]const u8) error{OutOfMemory}![]c
cmph_config_destroy(config);
const size = cmph_packed_size(mph);
var buf = try allocator.alloc(u8, size);
var buf = try allocator.alignedAlloc(u8, 8, size);
errdefer allocator.free(buf);
cmph_pack(mph, buf.ptr);
cmph_destroy(mph);
@ -43,13 +43,13 @@ pub fn pack(allocator: Allocator, input: [][*:0]const u8) error{OutOfMemory}![]c
}
// perfect-hash a list of numbers and return the packed mphf
pub fn packU32(allocator: Allocator, numbers: []const u32) error{OutOfMemory}![]const u8 {
var keys: [][6]u8 = try allocator.alloc([6]u8, numbers.len);
pub fn packU32(allocator: Allocator, numbers: []const u32) error{OutOfMemory}![]align(8) const u8 {
var keys: [][6]u8 = try allocator.alignedAlloc([6]u8, 8, numbers.len);
defer allocator.free(keys);
for (numbers) |n, i|
keys[i] = unzeroZ(n);
var keys2 = try allocator.alloc([*:0]const u8, numbers.len);
var keys2 = try allocator.alignedAlloc([*:0]const u8, 8, numbers.len);
defer allocator.free(keys2);
for (keys) |_, i|
keys2[i] = @ptrCast([*:0]const u8, &keys[i]);
@ -57,10 +57,10 @@ pub fn packU32(allocator: Allocator, numbers: []const u32) error{OutOfMemory}![]
}
// perfect-hash a list of strings and return the packed mphf
pub fn packStr(allocator: Allocator, strings: []const []const u8) error{OutOfMemory}![]const u8 {
pub fn packStr(allocator: Allocator, strings: []const []const u8) error{OutOfMemory}![]align(8) const u8 {
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
var keys = try arena.allocator().alloc([*:0]const u8, strings.len);
var keys = try arena.allocator().alignedAlloc([*:0]const u8, 8, strings.len);
for (strings) |_, i|
keys[i] = try arena.allocator().dupeZ(u8, strings[i]);
return pack(allocator, keys);

View File

@ -5,7 +5,7 @@
// golang's varint implementation.
const std = @import("std");
const ArrayList = std.ArrayList;
const ArrayListAligned = std.ArrayListAligned;
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const math = std.math;
@ -183,7 +183,7 @@ pub fn deltaDecompressionIterator(vit: *VarintSliceIterator) DeltaDecompressionI
};
}
pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void {
pub fn appendUvarint(arr: *ArrayListAligned(u8, 8), x: u64) Allocator.Error!void {
var buf: [maxVarintLen64]u8 = undefined;
const n = putUvarint(&buf, x);
try arr.appendSlice(buf[0..n]);
@ -221,7 +221,7 @@ test "compress putUvarint/uvarint" {
}
test "compress varintSliceIterator" {
var buf = ArrayList(u8).init(testing.allocator);
var buf = ArrayListAligned(u8, 8).init(testing.allocator);
defer buf.deinit();
try appendUvarint(&buf, uvarint_tests.len);
for (uvarint_tests) |x|
@ -245,7 +245,7 @@ test "compress delta compress/decompress" {
.{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } },
};
for (tests) |t| {
var arr = try ArrayList(u8).initCapacity(
var arr = try ArrayListAligned(u8, 8).initCapacity(
testing.allocator,
t.input.len,
);
@ -274,7 +274,7 @@ test "compress delta compression negative tests" {
&[_]u8{ 0, 1, 1 },
&[_]u8{ 0, 1, 2, 1 },
}) |t| {
var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
var arr = try ArrayListAligned(u8, 8).initCapacity(testing.allocator, t.len);
defer arr.deinit();
try arr.appendSlice(t);
try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items));
@ -286,7 +286,7 @@ test "compress delta decompress overflow" {
&[_]u8{ 255, 0 },
&[_]u8{ 0, 128, 127 },
}) |t| {
var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
var arr = try ArrayListAligned(u8, 8).initCapacity(testing.allocator, t.len);
defer arr.deinit();
try arr.appendSlice(t);
try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items));
@ -298,7 +298,7 @@ test "compress delta decompression with an iterator" {
std.mem.copy(u64, compressed[0..], uvarint_tests[0..]);
try deltaCompress(u64, compressed[0..]);
var buf = ArrayList(u8).init(testing.allocator);
var buf = ArrayListAligned(u8, 8).init(testing.allocator);
defer buf.deinit();
try appendUvarint(&buf, compressed.len);
for (compressed) |x|
@ -316,7 +316,7 @@ test "compress delta decompression with an iterator" {
test "compress appendUvarint" {
for (uvarint_tests) |x| {
var buf = ArrayList(u8).init(testing.allocator);
var buf = ArrayListAligned(u8, 8).init(testing.allocator);
defer buf.deinit();
try appendUvarint(&buf, x);

View File

@ -20,6 +20,15 @@ const Endian = enum(u4) {
}
};
pub const Host = packed struct {
endian: Endian = Endian.native(),
ptr_size: u4 = ptr_size,
pub fn new() Host {
return Host{};
}
};
pub const section_length_bits = 6;
pub const section_length = 1 << section_length_bits;
@ -30,28 +39,29 @@ pub const Invalid = error{
InvalidPointerSize,
};
pub const Header = packed struct {
magic: [4]u8 = magic,
version: u8 = version,
endian: Endian = Endian.native(),
ptr_size: u4 = ptr_size,
nblocks_shell_blob: u8,
num_shells: u8,
num_groups: u32,
num_users: u32,
nblocks_bdz_gid: u32,
nblocks_bdz_groupname: u32,
nblocks_bdz_uid: u32,
nblocks_bdz_username: u32,
pub const Header = extern struct {
magic: [8]u8 = magic ++ [1]u8{0} ** 4,
nblocks_groups: u64,
nblocks_users: u64,
nblocks_groupmembers: u64,
nblocks_additional_gids: u64,
getgr_bufsize: u64,
getpw_bufsize: u64,
padding: [48]u8 = [1]u8{0} ** 48,
pub fn fromBytes(blob: *const [@sizeOf(Header)]u8) Invalid!*const Header {
num_groups: u32,
num_users: u32,
nblocks_bdz_gid: u32,
nblocks_bdz_groupname: u32,
nblocks_bdz_uid: u32,
nblocks_bdz_username: u32,
nblocks_shell_blob: u8,
num_shells: u8,
version: u8 = version,
host: Host = Host.new(),
padding: [40]u8 = [1]u8{0} ** 40,
pub fn fromBytes(blob: *align(8) const [@sizeOf(Header)]u8) Invalid!*const Header {
const self = mem.bytesAsValue(Header, blob);
if (!mem.eql(u8, magic[0..4], blob[0..4]))
@ -60,13 +70,13 @@ pub const Header = packed struct {
if (self.version != 0)
return error.InvalidVersion;
if (self.endian != Endian.native())
if (self.host.endian != Endian.native())
return error.InvalidEndianess;
// when ptr size is larger than on the host that constructed it the DB,
// getgr_bufsize/getpw_bufsize may return insufficient values, causing
// OutOfMemory for getgr* and getpw* calls.
if (self.ptr_size < ptr_size)
if (self.host.ptr_size < ptr_size)
return error.InvalidPointerSize;
return self;
@ -80,7 +90,7 @@ test "header Section length is a power of two" {
}
test "header fits into two sections" {
try testing.expect(@sizeOf(Header) == 2 * section_length);
try testing.expectEqual(2 * section_length, @sizeOf(Header));
}
test "header bit header size is equal to @sizeOf(Header)" {

View File

@ -29,7 +29,7 @@ const ENV_DB = "TURBONSS_DB";
const ENV_VERBOSE = "TURBONSS_VERBOSE";
const ENV_OMIT_MEMBERS = "TURBONSS_OMIT_MEMBERS";
export var turbonss_db_path: [:0]const u8 = "/etc/turbonss/db.turbo";
export var turbonss_db_path: [*:0]const u8 = "/etc/turbonss/db.turbo";
// State is a type of the global variable holding the process state:
// the DB handle and all the iterators.
@ -90,7 +90,7 @@ fn init() void {
// argv does not exist because
// https://github.com/ziglang/zig/issues/4524#issuecomment-1184748756
// so reading /proc/self/cmdline because
// so reading /proc/self/cmdline
const fd = os.openZ("/proc/self/cmdline", os.O.RDONLY, 0) catch break :blk false;
defer os.close(fd);
break :blk isId(fd);
@ -99,7 +99,7 @@ fn init() void {
const fname = if (getenv(ENV_DB)) |env|
mem.sliceTo(env, 0)
else
turbonss_db_path;
mem.sliceTo(turbonss_db_path, 0);
const file = File.open(fname) catch |err| {
if (verbose)

View File

@ -1,7 +1,7 @@
const std = @import("std");
const assert = std.debug.assert;
const Allocator = std.mem.Allocator;
const ArrayList = std.ArrayList;
const ArrayListAligned = std.ArrayListAligned;
// rounds up an int to the nearest factor of nbits.
pub fn roundUp(comptime T: type, comptime nbits: u8, n: T) T {
@ -16,8 +16,8 @@ pub fn until(comptime T: type, comptime nbits: u8, n: T) T {
return roundUp(T, nbits, n) - n;
}
// arrayList adds padding to an ArrayList(u8) for a given number of nbits
pub fn arrayList(arr: *ArrayList(u8), comptime nbits: u8) Allocator.Error!void {
// arrayList adds padding to an ArrayListAligned(u8, 8) for a given number of nbits
pub fn arrayList(arr: *ArrayListAligned(u8, 8), comptime nbits: u8) Allocator.Error!void {
const padding = until(u64, nbits, arr.items.len);
try arr.*.appendNTimes(0, padding);
}
@ -40,7 +40,7 @@ test "padding" {
}
test "padding arrayList" {
var buf = try ArrayList(u8).initCapacity(testing.allocator, 16);
var buf = try ArrayListAligned(u8, 8).initCapacity(testing.allocator, 16);
defer buf.deinit();
buf.appendAssumeCapacity(1);

View File

@ -10,10 +10,10 @@ pub const max_shell_len = 256;
// ShellReader interprets "Shell Index" and "Shell Blob" sections.
pub const ShellReader = struct {
index: []const u16,
index: []align(8) const u16,
blob: []const u8,
pub fn init(index: []align