motiejus
/
turbonss
1
Fork 0
Code Releases Activity

delta decompression iterator

This commit is contained in:
Motiejus Jakštys 2022-03-06 06:29:16 +02:00 committed by Motiejus Jakštys
parent ec95b231fa
commit 039aeb80dd
1 changed files with 107 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
src/compress.zig
View File

@ -8,6 +8,7 @@ const std = @import("std");
const ArrayList = std.ArrayList;
const Allocator = std.mem.Allocator;
const assert = std.debug.assert;
const math = std.math;
// compresses a strictly incrementing sorted slice of integers using delta
// compression. Compression is in-place.
@ -35,8 +36,8 @@ pub fn deltaDecompress(comptime T: type, elems: []T) error{Overflow}!void {
var i: usize = 1;
while (i < elems.len) : (i += 1) {
const x = try std.math.add(T, elems[i - 1], 1);
elems[i] = try std.math.add(T, elems[i], x);
const x = try math.add(T, elems[i - 1], 1);
elems[i] = try math.add(T, elems[i], x);
}
}
@ -70,7 +71,7 @@ pub fn uvarint(buf: []const u8) error{Overflow}!Varint {
};
}
x |= (@as(u64, b & 0x7f) << s);
s = try std.math.add(u6, s, 7);
s = try math.add(u6, s, 7);
}
return Varint{
@ -113,7 +114,6 @@ const varintSliceIterator = struct {
pub fn VarintSliceIterator(arr: []const u8) error{Overflow}!varintSliceIterator {
const firstnumber = try uvarint(arr);
return varintSliceIterator{
.remaining = firstnumber.value,
.arr = arr,
@ -121,6 +121,31 @@ pub fn VarintSliceIterator(arr: []const u8) error{Overflow}!varintSliceIterator
};
}
const deltaDecompressionIterator = struct {
vit: *varintSliceIterator,
prev: u64,
add_to_prev: u1,
pub fn next(it: *deltaDecompressionIterator) error{Overflow}!?u64 {
const current = try it.vit.next();
if (current == null) return null;
const prevExtra = try math.add(u64, it.prev, it.add_to_prev);
const result = try math.add(u64, current.?, prevExtra);
it.prev = result;
it.add_to_prev = 1;
return result;
}
};
pub fn DeltaDecompressionIterator(vit: *varintSliceIterator) deltaDecompressionIterator {
return deltaDecompressionIterator{
.vit = vit,
.prev = 0,
.add_to_prev = 0,
};
}
pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void {
var buf: [maxVarintLen64]u8 = undefined;
const n = putUvarint(&buf, x);
@ -129,56 +154,6 @@ pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void {
const testing = std.testing;
test "delta compress/decompress" {
const tests = [_]struct { input: []const u8, want: []const u8 }{
.{ .input = &[_]u8{}, .want = &[_]u8{} },
.{ .input = &[_]u8{0}, .want = &[_]u8{0} },
.{ .input = &[_]u8{10}, .want = &[_]u8{10} },
.{ .input = &[_]u8{ 0, 1, 2 }, .want = &[_]u8{ 0, 0, 0 } },
.{ .input = &[_]u8{ 10, 20, 30, 255 }, .want = &[_]u8{ 10, 9, 9, 224 } },
.{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } },
};
for (tests) |t| {
var arr = try ArrayList(u8).initCapacity(
testing.allocator,
t.input.len,
);
defer arr.deinit();
try arr.appendSlice(t.input);
try deltaCompress(u8, arr.items);
try testing.expectEqualSlices(u8, arr.items, t.want);
try deltaDecompress(u8, arr.items);
try testing.expectEqualSlices(u8, arr.items, t.input);
}
}
test "delta compression negative tests" {
for ([_][]const u8{
&[_]u8{ 0, 0 },
&[_]u8{ 0, 1, 1 },
&[_]u8{ 0, 1, 2, 1 },
}) |t| {
var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
defer arr.deinit();
try arr.appendSlice(t);
try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items));
}
}
test "delta decompress overflow" {
for ([_][]const u8{
&[_]u8{ 255, 0 },
&[_]u8{ 0, 128, 127 },
}) |t| {
var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
defer arr.deinit();
try arr.appendSlice(t);
try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items));
}
}
const uvarint_tests = [_]u64{
0,
1,
@ -223,6 +198,84 @@ test "VarintSliceIterator" {
try testing.expectEqual(i, uvarint_tests.len);
}
test "delta compress/decompress" {
const tests = [_]struct { input: []const u8, want: []const u8 }{
.{ .input = &[_]u8{}, .want = &[_]u8{} },
.{ .input = &[_]u8{0}, .want = &[_]u8{0} },
.{ .input = &[_]u8{10}, .want = &[_]u8{10} },
.{ .input = &[_]u8{ 0, 1, 2 }, .want = &[_]u8{ 0, 0, 0 } },
.{ .input = &[_]u8{ 10, 20, 30, 255 }, .want = &[_]u8{ 10, 9, 9, 224 } },
.{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } },
};
for (tests) |t| {
var arr = try ArrayList(u8).initCapacity(
testing.allocator,
t.input.len,
);
defer arr.deinit();
try arr.appendSlice(t.input);
try deltaCompress(u8, arr.items);
try testing.expectEqualSlices(u8, arr.items, t.want);
try deltaDecompress(u8, arr.items);
try testing.expectEqualSlices(u8, arr.items, t.input);
}
}
test "delta compression with varint tests" {
var scratch: [uvarint_tests.len]u64 = undefined;
std.mem.copy(u64, scratch[0..], uvarint_tests[0..]);
try deltaCompress(u64, scratch[0..]);
try deltaDecompress(u64, scratch[0..]);
try testing.expectEqualSlices(u64, uvarint_tests[0..], scratch[0..]);
}
test "delta compression negative tests" {
for ([_][]const u8{
&[_]u8{ 0, 0 },
&[_]u8{ 0, 1, 1 },
&[_]u8{ 0, 1, 2, 1 },
}) |t| {
var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
defer arr.deinit();
try arr.appendSlice(t);
try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items));
}
}
test "delta decompress overflow" {
for ([_][]const u8{
&[_]u8{ 255, 0 },
&[_]u8{ 0, 128, 127 },
}) |t| {
var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len);
defer arr.deinit();
try arr.appendSlice(t);
try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items));
}
}
test "delta decompression with an iterator" {
const allocator = testing.allocator;
var compressed = try testing.allocator.dupe(u64, uvarint_tests[0..]);
defer allocator.free(compressed);
try deltaCompress(u64, compressed);
var buf = ArrayList(u8).init(testing.allocator);
defer buf.deinit();
try appendUvarint(&buf, compressed.len);
for (compressed) |x|
try appendUvarint(&buf, x);
var it = DeltaDecompressionIterator(&try VarintSliceIterator(buf.items));
var i: usize = 0;
while (try it.next()) |got| : (i += 1) {
try testing.expectEqual(uvarint_tests[i], got);
}
try testing.expectEqual(i, uvarint_tests.len);
}
test "appendUvarint" {
for (uvarint_tests) |x| {
var buf = ArrayList(u8).init(testing.allocator);