motiejus
/
jgit
1
Fork 0
Code Releases Activity

DFSGarbargeCollector: Write object size indices 

PackWriter knows how to add an object size index to the pack, but the
garbage collector is not using it yet.

Teach DfsGarbageCollector to write the object size index on
writePack(). Disable by default in the unreachable-garbage pack.

Callers control the content/presence of the index through the
PackConfig option (minBytesForObjSizeIndex) for all other packs, so
there is no need of a specific flag in DfsGarbageCollector.

Change-Id: I86f5f17310e6913381125bec4caab32dc45b7c9d
This commit is contained in:
Ivan Frade 2021-12-29 15:39:41 -08:00
parent 9dace2e6d6
commit 12a4a4ccaa
2 changed files with 83 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
View File

@ -15,6 +15,7 @@
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
@ -1100,6 +1101,70 @@ public void commitGraphWithoutGCrestPack() throws Exception {
}
}
@Test
public void objectSizeIdx_reachableBlob_bigEnough_indexed() throws Exception {
String master = "refs/heads/master";
RevCommit root = git.branch(master).commit().message("root").noParents()
.create();
RevBlob headsBlob = git.blob("twelve bytes");
git.branch(master).commit()
.message("commit on head")
.add("file.txt", headsBlob)
.parent(root)
.create();
gcWithObjectSizeIndex(10);
DfsReader reader = odb.newReader();
DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC);
assertTrue(gcPack.hasObjectSizeIndex(reader));
assertEquals(12, gcPack.getIndexedObjectSize(reader, headsBlob));
}
@Test
public void objectSizeIdx_reachableBlob_tooSmall_notIndexed() throws Exception {
String master = "refs/heads/master";
RevCommit root = git.branch(master).commit().message("root").noParents()
.create();
RevBlob tooSmallBlob = git.blob("small");
git.branch(master).commit()
.message("commit on head")
.add("small.txt", tooSmallBlob)
.parent(root)
.create();
gcWithObjectSizeIndex(10);
DfsReader reader = odb.newReader();
DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC);
assertTrue(gcPack.hasObjectSizeIndex(reader));
assertEquals(-1, gcPack.getIndexedObjectSize(reader, tooSmallBlob));
}
@Test
public void objectSizeIndex_unreachableGarbage_noIdx() throws Exception {
String master = "refs/heads/master";
RevCommit root = git.branch(master).commit().message("root").noParents()
.create();
git.branch(master).commit()
.message("commit on head")
.add("file.txt", git.blob("a blob"))
.parent(root)
.create();
git.update(master, root); // blob is unreachable
gcWithObjectSizeIndex(0);
DfsReader reader = odb.newReader();
DfsPackFile gcRestPack = findFirstBySource(odb.getPacks(), UNREACHABLE_GARBAGE);
assertFalse(gcRestPack.hasObjectSizeIndex(reader));
}
private static DfsPackFile findFirstBySource(DfsPackFile[] packs, PackSource source) {
return Arrays.stream(packs)
.filter(p -> p.getPackDescription().getPackSource() == source)
.findFirst().get();
}
private TestRepository<InMemoryRepository>.CommitBuilder commit() {
return git.commit();
}
@ -1110,6 +1175,12 @@ private void gcWithCommitGraph() throws IOException {
run(gc);
}
private void gcWithObjectSizeIndex(int threshold) throws IOException {
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
gc.getPackConfig().setMinBytesForObjSizeIndex(threshold);
run(gc);
}
private void gcNoTtl() throws IOException {
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL

12
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java
View File

@ -20,6 +20,7 @@
import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
@ -681,6 +682,17 @@ private DfsPackDescription writePack(PackSource source, PackWriter pw,
pack.setIndexVersion(pw.getIndexVersion());
}
if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) {
try (DfsOutputStream out = objdb.writeFile(pack,
OBJECT_SIZE_INDEX)) {
CountingOutputStream cnt = new CountingOutputStream(out);
pw.writeObjectSizeIndex(cnt);
pack.addFileExt(OBJECT_SIZE_INDEX);
pack.setFileSize(OBJECT_SIZE_INDEX, cnt.getCount());
pack.setBlockSize(OBJECT_SIZE_INDEX, out.blockSize());
}
}
if (pw.prepareBitmapIndex(pm)) {
try (DfsOutputStream out = objdb.writeFile(pack, BITMAP_INDEX)) {
CountingOutputStream cnt = new CountingOutputStream(out);