motiejus
/
turbonss
1
Fork 0
Code Releases Activity

expose native shell sections

main
Motiejus Jakštys 2022-03-05 06:08:01 +02:00 committed by Motiejus Jakštys
parent 869f3f3a65
commit e17b2c9641
3 changed files with 45 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
README.md
View File

@ -349,23 +349,23 @@ Complete file structure
Each section is padded to 64 bytes.
```
STATUS SECTION SIZE DESCRIPTION
✅ Header 48 see "Turbonss header" section
✅ bdz_gid ? bdz(gid)
✅ bdz_groupname ? bdz(groupname)
✅ bdz_uid ? bdz(uid)
✅ bdz_username ? bdz(username)
idx_gid2group len(group)*29/8 bdz->offset Groups
idx_groupname2group len(group)*29/8 bdz->offset Groups
idx_uid2user len(user)*29/8 bdz->offset Users
idx_name2user len(user)*29/8 bdz->offset Users
idx_username2gids len(user)*29/8 bdz->offset UserGids
✅ shellIndex len(shells)*2 shell index array
✅ shellBlob <= 4032 shell data blob (max 63*64 bytes)
✅ groups ? packed Group entries (8b padding)
✅ users ? packed User entries (8b padding)
groupMembers ? per-group memberlist (no padding)
userGids ? per-user gidlist entries (8b padding)
SECTION SIZE DESCRIPTION
Header 48 see "Turbonss header" section
bdz_gid ? bdz(gid)
bdz_groupname ? bdz(groupname)
bdz_uid ? bdz(uid)
bdz_username ? bdz(username)
idx_gid2group len(group)*29/8 bdz->offset Groups
idx_groupname2group len(group)*29/8 bdz->offset Groups
idx_uid2user len(user)*29/8 bdz->offset Users
idx_name2user len(user)*29/8 bdz->offset Users
idx_username2gids len(user)*29/8 bdz->offset UserGids
shellIndex len(shells)*2 shell index array
shellBlob <= 4032 shell data blob (max 63*64 bytes)
groups ? packed Group entries (8b padding)
users ? packed User entries (8b padding)
groupMembers ? per-group memberlist (no padding)
userGids ? per-user gidlist entries (8b padding)
```
Section creation order:
@ -373,7 +373,7 @@ Section creation order:
1. ✅ `bdz_*`. No depdendencies.
1. ✅ `shellIndex`, `shellBlob`. No dependencies.
1. ✅ userGids. No dependencies.
1. Users. Requires `userGids`.
1. Users. Requires `userGids` and shell.
1. Groupmembers. Requires Users.
1. Groups. Requires Groupmembers.
1. `idx_*`. Requires offsets to Groups and Users.

36
src/sections.zig
View File

@ -20,6 +20,7 @@ const cmph = @import("cmph.zig");
const bdz = @import("bdz.zig");
const User = userImport.User;
const Group = groupImport.Group;
const ShellSections = shellImport.ShellWriter.ShellSections;
const Corpus = struct {
arena: std.heap.ArenaAllocator,
@ -190,17 +191,6 @@ pub fn bdzUsername(allocator: Allocator, corpus: *const Corpus) cmph.Error![]con
return try cmph.pack_str(allocator, corpus.usersMulti.items(.name));
}
pub const ShellSections = struct {
index: []const u8,
blob: []const u8,
pub fn deinit(self: *ShellSections, allocator: Allocator) void {
allocator.free(self.index);
allocator.free(self.blob);
self.* = undefined;
}
};
// TODO(motiejus) there are a few problems:
// - memory management for shell sections is a mess. Make it easier by ...
// - shell module should accept a list of shells and spit out two slices
@ -212,17 +202,10 @@ pub fn shellSections(
corpus: *const Corpus,
) shellSectionsErr!ShellSections {
var popcon = shellImport.ShellWriter.init(allocator);
defer popcon.deinit();
for (corpus.usersMulti.items(.shell)) |shell| {
try popcon.put(shell);
}
var sections = try popcon.toOwnedSections(shellImport.max_shells);
defer sections.deinit();
return ShellSections{
.index = try allocator.dupe(u8, sections.sectionIndex()),
.blob = try allocator.dupe(u8, sections.sectionBlob()),
};
return popcon.toOwnedSections(shellImport.max_shells);
}
pub const UserGids = struct {
@ -276,6 +259,19 @@ pub fn userGids(allocator: Allocator, corpus: *const Corpus) userGidsErr!UserGid
};
}
pub const usersSectionErr = Allocator.Error || error{Overflow};
pub fn usersSection(
allocator: Allocator,
corpus: *const Corpus,
gids: UserGids,
shells: ShellSections,
) usersSectionErr![]const u8 {
_ = allocator;
_ = corpus;
_ = gids;
_ = shells;
}
pub fn groupMembers(allocator: Allocator, corpus: *const Corpus) Allocator.Error!void {
var buf: [compress.maxVarintLen64]u8 = undefined;
var offsets = ArrayListUnmanaged(usize).initCapacity(
@ -447,7 +443,7 @@ test "test sections" {
defer allocator.free(bdz_username);
var shell_sections = try shellSections(allocator, &corpus);
defer shell_sections.deinit(allocator);
defer shell_sections.deinit();
var user_gids = try userGids(allocator, &corpus);
defer user_gids.deinit(allocator);

27
src/shell.zig
View File

@ -54,7 +54,7 @@ pub const ShellWriter = struct {
score: u64,
};
const ShellSections = struct {
pub const ShellSections = struct {
index: BoundedArray(ShellIndex, max_shells),
blob: BoundedArray(u8, max_shells * max_shell_len),
indices: StringHashMap(u6),
@ -71,15 +71,16 @@ pub const ShellWriter = struct {
.blob = try BoundedArray(u8, max_shells * max_shell_len).init(0),
.indices = StringHashMap(u6).init(allocator),
};
errdefer self.indices.deinit();
var fullOffset: u12 = 0;
var idx: u6 = 0;
while (idx < shells.len) : (idx += 1) {
const len = @intCast(u6, shells.get(idx).len);
const len = try std.math.cast(u6, shells.get(idx).len);
try self.blob.appendSlice(shells.get(idx));
const ourShell = self.blob.constSlice()[fullOffset .. fullOffset + len];
try self.indices.put(ourShell, idx);
self.index.set(idx, ShellIndex{
.offset = @intCast(u10, fullOffset >> 2),
.offset = try std.math.cast(u10, fullOffset >> 2),
.len = len - 1,
});
@ -118,26 +119,23 @@ pub const ShellWriter = struct {
pub fn deinit(self: *ShellWriter) void {
var it = self.counts.keyIterator();
while (it.next()) |key_ptr| {
while (it.next()) |key_ptr|
self.counts.allocator.free(key_ptr.*);
}
self.counts.deinit();
self.* = undefined;
}
pub fn put(self: *ShellWriter, shell: []const u8) !void {
const res = try self.counts.getOrPutAdapted(shell, self.counts.ctx);
if (res.found_existing) {
res.value_ptr.* += 1;
} else {
if (!res.found_existing) {
res.key_ptr.* = try self.allocator.dupe(u8, shell);
res.value_ptr.* = 1;
} else {
res.value_ptr.* += 1;
}
}
fn cmpShells(context: void, a: KV, b: KV) std.math.Order {
_ = context;
fn cmpShells(_: void, a: KV, b: KV) std.math.Order {
return std.math.order(a.score, b.score);
}
@ -161,10 +159,8 @@ pub const ShellWriter = struct {
var topShells = try BoundedArray([]const u8, max_shells).init(total);
var i: u32 = 0;
while (i < total) : (i += 1) {
const elem = deque.removeMax().shell;
topShells.set(i, elem);
}
while (i < total) : (i += 1)
topShells.set(i, deque.removeMax().shell);
const result = ShellSections.init(self.allocator, topShells);
const allocator = self.allocator;
@ -178,7 +174,6 @@ const testing = std.testing;
test "basic shellpopcon" {
var popcon = ShellWriter.init(testing.allocator);
defer popcon.deinit();
const bash = "/bin/bash"; // 9 chars
const zsh = "/bin/zsh"; // 8 chars