From 039aeb80ddb51fe8045353c6f7f8ce72d9e72bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Sun, 6 Mar 2022 06:29:16 +0200 Subject: [PATCH] delta decompression iterator --- src/compress.zig | 161 +++++++++++++++++++++++++++++++---------------- 1 file changed, 107 insertions(+), 54 deletions(-) diff --git a/src/compress.zig b/src/compress.zig index 396a623..92e24ff 100644 --- a/src/compress.zig +++ b/src/compress.zig @@ -8,6 +8,7 @@ const std = @import("std"); const ArrayList = std.ArrayList; const Allocator = std.mem.Allocator; const assert = std.debug.assert; +const math = std.math; // compresses a strictly incrementing sorted slice of integers using delta // compression. Compression is in-place. @@ -35,8 +36,8 @@ pub fn deltaDecompress(comptime T: type, elems: []T) error{Overflow}!void { var i: usize = 1; while (i < elems.len) : (i += 1) { - const x = try std.math.add(T, elems[i - 1], 1); - elems[i] = try std.math.add(T, elems[i], x); + const x = try math.add(T, elems[i - 1], 1); + elems[i] = try math.add(T, elems[i], x); } } @@ -70,7 +71,7 @@ pub fn uvarint(buf: []const u8) error{Overflow}!Varint { }; } x |= (@as(u64, b & 0x7f) << s); - s = try std.math.add(u6, s, 7); + s = try math.add(u6, s, 7); } return Varint{ @@ -113,7 +114,6 @@ const varintSliceIterator = struct { pub fn VarintSliceIterator(arr: []const u8) error{Overflow}!varintSliceIterator { const firstnumber = try uvarint(arr); - return varintSliceIterator{ .remaining = firstnumber.value, .arr = arr, @@ -121,6 +121,31 @@ pub fn VarintSliceIterator(arr: []const u8) error{Overflow}!varintSliceIterator }; } +const deltaDecompressionIterator = struct { + vit: *varintSliceIterator, + prev: u64, + add_to_prev: u1, + + pub fn next(it: *deltaDecompressionIterator) error{Overflow}!?u64 { + const current = try it.vit.next(); + if (current == null) return null; + + const prevExtra = try math.add(u64, it.prev, it.add_to_prev); + const result = try math.add(u64, current.?, prevExtra); + it.prev = result; + it.add_to_prev = 1; + return result; + } +}; + +pub fn DeltaDecompressionIterator(vit: *varintSliceIterator) deltaDecompressionIterator { + return deltaDecompressionIterator{ + .vit = vit, + .prev = 0, + .add_to_prev = 0, + }; +} + pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void { var buf: [maxVarintLen64]u8 = undefined; const n = putUvarint(&buf, x); @@ -129,56 +154,6 @@ pub fn appendUvarint(arr: *ArrayList(u8), x: u64) Allocator.Error!void { const testing = std.testing; -test "delta compress/decompress" { - const tests = [_]struct { input: []const u8, want: []const u8 }{ - .{ .input = &[_]u8{}, .want = &[_]u8{} }, - .{ .input = &[_]u8{0}, .want = &[_]u8{0} }, - .{ .input = &[_]u8{10}, .want = &[_]u8{10} }, - .{ .input = &[_]u8{ 0, 1, 2 }, .want = &[_]u8{ 0, 0, 0 } }, - .{ .input = &[_]u8{ 10, 20, 30, 255 }, .want = &[_]u8{ 10, 9, 9, 224 } }, - .{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } }, - }; - for (tests) |t| { - var arr = try ArrayList(u8).initCapacity( - testing.allocator, - t.input.len, - ); - defer arr.deinit(); - try arr.appendSlice(t.input); - - try deltaCompress(u8, arr.items); - try testing.expectEqualSlices(u8, arr.items, t.want); - - try deltaDecompress(u8, arr.items); - try testing.expectEqualSlices(u8, arr.items, t.input); - } -} - -test "delta compression negative tests" { - for ([_][]const u8{ - &[_]u8{ 0, 0 }, - &[_]u8{ 0, 1, 1 }, - &[_]u8{ 0, 1, 2, 1 }, - }) |t| { - var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len); - defer arr.deinit(); - try arr.appendSlice(t); - try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items)); - } -} - -test "delta decompress overflow" { - for ([_][]const u8{ - &[_]u8{ 255, 0 }, - &[_]u8{ 0, 128, 127 }, - }) |t| { - var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len); - defer arr.deinit(); - try arr.appendSlice(t); - try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items)); - } -} - const uvarint_tests = [_]u64{ 0, 1, @@ -223,6 +198,84 @@ test "VarintSliceIterator" { try testing.expectEqual(i, uvarint_tests.len); } +test "delta compress/decompress" { + const tests = [_]struct { input: []const u8, want: []const u8 }{ + .{ .input = &[_]u8{}, .want = &[_]u8{} }, + .{ .input = &[_]u8{0}, .want = &[_]u8{0} }, + .{ .input = &[_]u8{10}, .want = &[_]u8{10} }, + .{ .input = &[_]u8{ 0, 1, 2 }, .want = &[_]u8{ 0, 0, 0 } }, + .{ .input = &[_]u8{ 10, 20, 30, 255 }, .want = &[_]u8{ 10, 9, 9, 224 } }, + .{ .input = &[_]u8{ 0, 254, 255 }, .want = &[_]u8{ 0, 253, 0 } }, + }; + for (tests) |t| { + var arr = try ArrayList(u8).initCapacity( + testing.allocator, + t.input.len, + ); + defer arr.deinit(); + try arr.appendSlice(t.input); + + try deltaCompress(u8, arr.items); + try testing.expectEqualSlices(u8, arr.items, t.want); + + try deltaDecompress(u8, arr.items); + try testing.expectEqualSlices(u8, arr.items, t.input); + } +} + +test "delta compression with varint tests" { + var scratch: [uvarint_tests.len]u64 = undefined; + std.mem.copy(u64, scratch[0..], uvarint_tests[0..]); + try deltaCompress(u64, scratch[0..]); + try deltaDecompress(u64, scratch[0..]); + try testing.expectEqualSlices(u64, uvarint_tests[0..], scratch[0..]); +} + +test "delta compression negative tests" { + for ([_][]const u8{ + &[_]u8{ 0, 0 }, + &[_]u8{ 0, 1, 1 }, + &[_]u8{ 0, 1, 2, 1 }, + }) |t| { + var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len); + defer arr.deinit(); + try arr.appendSlice(t); + try testing.expectError(error.NotSorted, deltaCompress(u8, arr.items)); + } +} + +test "delta decompress overflow" { + for ([_][]const u8{ + &[_]u8{ 255, 0 }, + &[_]u8{ 0, 128, 127 }, + }) |t| { + var arr = try ArrayList(u8).initCapacity(testing.allocator, t.len); + defer arr.deinit(); + try arr.appendSlice(t); + try testing.expectError(error.Overflow, deltaDecompress(u8, arr.items)); + } +} + +test "delta decompression with an iterator" { + const allocator = testing.allocator; + var compressed = try testing.allocator.dupe(u64, uvarint_tests[0..]); + defer allocator.free(compressed); + try deltaCompress(u64, compressed); + + var buf = ArrayList(u8).init(testing.allocator); + defer buf.deinit(); + try appendUvarint(&buf, compressed.len); + for (compressed) |x| + try appendUvarint(&buf, x); + + var it = DeltaDecompressionIterator(&try VarintSliceIterator(buf.items)); + var i: usize = 0; + while (try it.next()) |got| : (i += 1) { + try testing.expectEqual(uvarint_tests[i], got); + } + try testing.expectEqual(i, uvarint_tests.len); +} + test "appendUvarint" { for (uvarint_tests) |x| { var buf = ArrayList(u8).init(testing.allocator);