From 12a4a4ccaadefa7e2dd157ca2bd211ef460b825c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 29 Dec 2021 15:39:41 -0800 Subject: [PATCH] DFSGarbargeCollector: Write object size indices PackWriter knows how to add an object size index to the pack, but the garbage collector is not using it yet. Teach DfsGarbageCollector to write the object size index on writePack(). Disable by default in the unreachable-garbage pack. Callers control the content/presence of the index through the PackConfig option (minBytesForObjSizeIndex) for all other packs, so there is no need of a specific flag in DfsGarbageCollector. Change-Id: I86f5f17310e6913381125bec4caab32dc45b7c9d --- .../storage/dfs/DfsGarbageCollectorTest.java | 71 +++++++++++++++++++ .../storage/dfs/DfsGarbageCollector.java | 12 ++++ 2 files changed, 83 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index ab998951f..405e12677 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -15,6 +15,7 @@ import static org.junit.Assert.fail; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -1100,6 +1101,70 @@ public void commitGraphWithoutGCrestPack() throws Exception { } } + @Test + public void objectSizeIdx_reachableBlob_bigEnough_indexed() throws Exception { + String master = "refs/heads/master"; + RevCommit root = git.branch(master).commit().message("root").noParents() + .create(); + RevBlob headsBlob = git.blob("twelve bytes"); + git.branch(master).commit() + .message("commit on head") + .add("file.txt", headsBlob) + .parent(root) + .create(); + + gcWithObjectSizeIndex(10); + + DfsReader reader = odb.newReader(); + DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); + assertTrue(gcPack.hasObjectSizeIndex(reader)); + assertEquals(12, gcPack.getIndexedObjectSize(reader, headsBlob)); + } + + @Test + public void objectSizeIdx_reachableBlob_tooSmall_notIndexed() throws Exception { + String master = "refs/heads/master"; + RevCommit root = git.branch(master).commit().message("root").noParents() + .create(); + RevBlob tooSmallBlob = git.blob("small"); + git.branch(master).commit() + .message("commit on head") + .add("small.txt", tooSmallBlob) + .parent(root) + .create(); + + gcWithObjectSizeIndex(10); + + DfsReader reader = odb.newReader(); + DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); + assertTrue(gcPack.hasObjectSizeIndex(reader)); + assertEquals(-1, gcPack.getIndexedObjectSize(reader, tooSmallBlob)); + } + + @Test + public void objectSizeIndex_unreachableGarbage_noIdx() throws Exception { + String master = "refs/heads/master"; + RevCommit root = git.branch(master).commit().message("root").noParents() + .create(); + git.branch(master).commit() + .message("commit on head") + .add("file.txt", git.blob("a blob")) + .parent(root) + .create(); + git.update(master, root); // blob is unreachable + gcWithObjectSizeIndex(0); + + DfsReader reader = odb.newReader(); + DfsPackFile gcRestPack = findFirstBySource(odb.getPacks(), UNREACHABLE_GARBAGE); + assertFalse(gcRestPack.hasObjectSizeIndex(reader)); + } + + private static DfsPackFile findFirstBySource(DfsPackFile[] packs, PackSource source) { + return Arrays.stream(packs) + .filter(p -> p.getPackDescription().getPackSource() == source) + .findFirst().get(); + } + private TestRepository.CommitBuilder commit() { return git.commit(); } @@ -1110,6 +1175,12 @@ private void gcWithCommitGraph() throws IOException { run(gc); } + private void gcWithObjectSizeIndex(int threshold) throws IOException { + DfsGarbageCollector gc = new DfsGarbageCollector(repo); + gc.getPackConfig().setMinBytesForObjSizeIndex(threshold); + run(gc); + } + private void gcNoTtl() throws IOException { DfsGarbageCollector gc = new DfsGarbageCollector(repo); gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 92e23b8b4..9f415782f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -20,6 +20,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE; @@ -681,6 +682,17 @@ private DfsPackDescription writePack(PackSource source, PackWriter pw, pack.setIndexVersion(pw.getIndexVersion()); } + if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) { + try (DfsOutputStream out = objdb.writeFile(pack, + OBJECT_SIZE_INDEX)) { + CountingOutputStream cnt = new CountingOutputStream(out); + pw.writeObjectSizeIndex(cnt); + pack.addFileExt(OBJECT_SIZE_INDEX); + pack.setFileSize(OBJECT_SIZE_INDEX, cnt.getCount()); + pack.setBlockSize(OBJECT_SIZE_INDEX, out.blockSize()); + } + } + if (pw.prepareBitmapIndex(pm)) { try (DfsOutputStream out = objdb.writeFile(pack, BITMAP_INDEX)) { CountingOutputStream cnt = new CountingOutputStream(out);