expose native shell sections
This commit is contained in:
Motiejus Jakštys
parent
869f3f3a65
commit
e17b2c9641
36
README.md
36
README.md
@ -349,23 +349,23 @@ Complete file structure
|
||||
Each section is padded to 64 bytes.
|
||||
|
||||
```
|
||||
STATUS SECTION SIZE DESCRIPTION
|
||||
✅ Header 48 see "Turbonss header" section
|
||||
✅ bdz_gid ? bdz(gid)
|
||||
✅ bdz_groupname ? bdz(groupname)
|
||||
✅ bdz_uid ? bdz(uid)
|
||||
✅ bdz_username ? bdz(username)
|
||||
idx_gid2group len(group)*29/8 bdz->offset Groups
|
||||
idx_groupname2group len(group)*29/8 bdz->offset Groups
|
||||
idx_uid2user len(user)*29/8 bdz->offset Users
|
||||
idx_name2user len(user)*29/8 bdz->offset Users
|
||||
idx_username2gids len(user)*29/8 bdz->offset UserGids
|
||||
✅ shellIndex len(shells)*2 shell index array
|
||||
✅ shellBlob <= 4032 shell data blob (max 63*64 bytes)
|
||||
✅ groups ? packed Group entries (8b padding)
|
||||
✅ users ? packed User entries (8b padding)
|
||||
groupMembers ? per-group memberlist (no padding)
|
||||
userGids ? per-user gidlist entries (8b padding)
|
||||
SECTION SIZE DESCRIPTION
|
||||
Header 48 see "Turbonss header" section
|
||||
bdz_gid ? bdz(gid)
|
||||
bdz_groupname ? bdz(groupname)
|
||||
bdz_uid ? bdz(uid)
|
||||
bdz_username ? bdz(username)
|
||||
idx_gid2group len(group)*29/8 bdz->offset Groups
|
||||
idx_groupname2group len(group)*29/8 bdz->offset Groups
|
||||
idx_uid2user len(user)*29/8 bdz->offset Users
|
||||
idx_name2user len(user)*29/8 bdz->offset Users
|
||||
idx_username2gids len(user)*29/8 bdz->offset UserGids
|
||||
shellIndex len(shells)*2 shell index array
|
||||
shellBlob <= 4032 shell data blob (max 63*64 bytes)
|
||||
groups ? packed Group entries (8b padding)
|
||||
users ? packed User entries (8b padding)
|
||||
groupMembers ? per-group memberlist (no padding)
|
||||
userGids ? per-user gidlist entries (8b padding)
|
||||
```
|
||||
|
||||
Section creation order:
|
||||
@ -373,7 +373,7 @@ Section creation order:
|
||||
1. ✅ `bdz_*`. No depdendencies.
|
||||
1. ✅ `shellIndex`, `shellBlob`. No dependencies.
|
||||
1. ✅ userGids. No dependencies.
|
||||
1. Users. Requires `userGids`.
|
||||
1. Users. Requires `userGids` and shell.
|
||||
1. Groupmembers. Requires Users.
|
||||
1. Groups. Requires Groupmembers.
|
||||
1. `idx_*`. Requires offsets to Groups and Users.
|
||||
|
||||
@ -20,6 +20,7 @@ const cmph = @import("cmph.zig");
|
||||
const bdz = @import("bdz.zig");
|
||||
const User = userImport.User;
|
||||
const Group = groupImport.Group;
|
||||
const ShellSections = shellImport.ShellWriter.ShellSections;
|
||||
|
||||
const Corpus = struct {
|
||||
arena: std.heap.ArenaAllocator,
|
||||
@ -190,17 +191,6 @@ pub fn bdzUsername(allocator: Allocator, corpus: *const Corpus) cmph.Error![]con
|
||||
return try cmph.pack_str(allocator, corpus.usersMulti.items(.name));
|
||||
}
|
||||
|
||||
pub const ShellSections = struct {
|
||||
index: []const u8,
|
||||
blob: []const u8,
|
||||
|
||||
pub fn deinit(self: *ShellSections, allocator: Allocator) void {
|
||||
allocator.free(self.index);
|
||||
allocator.free(self.blob);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
// TODO(motiejus) there are a few problems:
|
||||
// - memory management for shell sections is a mess. Make it easier by ...
|
||||
// - shell module should accept a list of shells and spit out two slices
|
||||
@ -212,17 +202,10 @@ pub fn shellSections(
|
||||
corpus: *const Corpus,
|
||||
) shellSectionsErr!ShellSections {
|
||||
var popcon = shellImport.ShellWriter.init(allocator);
|
||||
defer popcon.deinit();
|
||||
for (corpus.usersMulti.items(.shell)) |shell| {
|
||||
try popcon.put(shell);
|
||||
}
|
||||
var sections = try popcon.toOwnedSections(shellImport.max_shells);
|
||||
defer sections.deinit();
|
||||
|
||||
return ShellSections{
|
||||
.index = try allocator.dupe(u8, sections.sectionIndex()),
|
||||
.blob = try allocator.dupe(u8, sections.sectionBlob()),
|
||||
};
|
||||
return popcon.toOwnedSections(shellImport.max_shells);
|
||||
}
|
||||
|
||||
pub const UserGids = struct {
|
||||
@ -276,6 +259,19 @@ pub fn userGids(allocator: Allocator, corpus: *const Corpus) userGidsErr!UserGid
|
||||
};
|
||||
}
|
||||
|
||||
pub const usersSectionErr = Allocator.Error || error{Overflow};
|
||||
pub fn usersSection(
|
||||
allocator: Allocator,
|
||||
corpus: *const Corpus,
|
||||
gids: UserGids,
|
||||
shells: ShellSections,
|
||||
) usersSectionErr![]const u8 {
|
||||
_ = allocator;
|
||||
_ = corpus;
|
||||
_ = gids;
|
||||
_ = shells;
|
||||
}
|
||||
|
||||
pub fn groupMembers(allocator: Allocator, corpus: *const Corpus) Allocator.Error!void {
|
||||
var buf: [compress.maxVarintLen64]u8 = undefined;
|
||||
var offsets = ArrayListUnmanaged(usize).initCapacity(
|
||||
@ -447,7 +443,7 @@ test "test sections" {
|
||||
defer allocator.free(bdz_username);
|
||||
|
||||
var shell_sections = try shellSections(allocator, &corpus);
|
||||
defer shell_sections.deinit(allocator);
|
||||
defer shell_sections.deinit();
|
||||
|
||||
var user_gids = try userGids(allocator, &corpus);
|
||||
defer user_gids.deinit(allocator);
|
||||
|
||||
@ -54,7 +54,7 @@ pub const ShellWriter = struct {
|
||||
score: u64,
|
||||
};
|
||||
|
||||
const ShellSections = struct {
|
||||
pub const ShellSections = struct {
|
||||
index: BoundedArray(ShellIndex, max_shells),
|
||||
blob: BoundedArray(u8, max_shells * max_shell_len),
|
||||
indices: StringHashMap(u6),
|
||||
@ -71,15 +71,16 @@ pub const ShellWriter = struct {
|
||||
.blob = try BoundedArray(u8, max_shells * max_shell_len).init(0),
|
||||
.indices = StringHashMap(u6).init(allocator),
|
||||
};
|
||||
errdefer self.indices.deinit();
|
||||
var fullOffset: u12 = 0;
|
||||
var idx: u6 = 0;
|
||||
while (idx < shells.len) : (idx += 1) {
|
||||
const len = @intCast(u6, shells.get(idx).len);
|
||||
const len = try std.math.cast(u6, shells.get(idx).len);
|
||||
try self.blob.appendSlice(shells.get(idx));
|
||||
const ourShell = self.blob.constSlice()[fullOffset .. fullOffset + len];
|
||||
try self.indices.put(ourShell, idx);
|
||||
self.index.set(idx, ShellIndex{
|
||||
.offset = @intCast(u10, fullOffset >> 2),
|
||||
.offset = try std.math.cast(u10, fullOffset >> 2),
|
||||
.len = len - 1,
|
||||
});
|
||||
|
||||
@ -118,26 +119,23 @@ pub const ShellWriter = struct {
|
||||
|
||||
pub fn deinit(self: *ShellWriter) void {
|
||||
var it = self.counts.keyIterator();
|
||||
while (it.next()) |key_ptr| {
|
||||
while (it.next()) |key_ptr|
|
||||
self.counts.allocator.free(key_ptr.*);
|
||||
}
|
||||
self.counts.deinit();
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn put(self: *ShellWriter, shell: []const u8) !void {
|
||||
const res = try self.counts.getOrPutAdapted(shell, self.counts.ctx);
|
||||
|
||||
if (res.found_existing) {
|
||||
res.value_ptr.* += 1;
|
||||
} else {
|
||||
if (!res.found_existing) {
|
||||
res.key_ptr.* = try self.allocator.dupe(u8, shell);
|
||||
res.value_ptr.* = 1;
|
||||
} else {
|
||||
res.value_ptr.* += 1;
|
||||
}
|
||||
}
|
||||
|
||||
fn cmpShells(context: void, a: KV, b: KV) std.math.Order {
|
||||
_ = context;
|
||||
fn cmpShells(_: void, a: KV, b: KV) std.math.Order {
|
||||
return std.math.order(a.score, b.score);
|
||||
}
|
||||
|
||||
@ -161,10 +159,8 @@ pub const ShellWriter = struct {
|
||||
var topShells = try BoundedArray([]const u8, max_shells).init(total);
|
||||
|
||||
var i: u32 = 0;
|
||||
while (i < total) : (i += 1) {
|
||||
const elem = deque.removeMax().shell;
|
||||
topShells.set(i, elem);
|
||||
}
|
||||
while (i < total) : (i += 1)
|
||||
topShells.set(i, deque.removeMax().shell);
|
||||
|
||||
const result = ShellSections.init(self.allocator, topShells);
|
||||
const allocator = self.allocator;
|
||||
@ -178,7 +174,6 @@ const testing = std.testing;
|
||||
|
||||
test "basic shellpopcon" {
|
||||
var popcon = ShellWriter.init(testing.allocator);
|
||||
defer popcon.deinit();
|
||||
|
||||
const bash = "/bin/bash"; // 9 chars
|
||||
const zsh = "/bin/zsh"; // 8 chars
|
||||
|
||||
Loading…
Reference in New Issue
Block a user