Merge changes from topic 'dfs-reftable'
* changes: dfs: write reftable from DfsGarbageCollector dfs: compact reftables during DfsPackCompactor
This commit is contained in:
commit
d26309c4d9
|
@ -5,6 +5,7 @@
|
||||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
|
||||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
||||||
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertNotNull;
|
import static org.junit.Assert.assertNotNull;
|
||||||
|
@ -13,19 +14,29 @@
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
|
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.RefCursor;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
|
||||||
import org.eclipse.jgit.junit.MockSystemReader;
|
import org.eclipse.jgit.junit.MockSystemReader;
|
||||||
import org.eclipse.jgit.junit.TestRepository;
|
import org.eclipse.jgit.junit.TestRepository;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
|
import org.eclipse.jgit.lib.BatchRefUpdate;
|
||||||
|
import org.eclipse.jgit.lib.NullProgressMonitor;
|
||||||
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
|
import org.eclipse.jgit.lib.ObjectIdRef;
|
||||||
import org.eclipse.jgit.lib.Ref;
|
import org.eclipse.jgit.lib.Ref;
|
||||||
import org.eclipse.jgit.lib.Repository;
|
import org.eclipse.jgit.lib.Repository;
|
||||||
import org.eclipse.jgit.revwalk.RevBlob;
|
import org.eclipse.jgit.revwalk.RevBlob;
|
||||||
import org.eclipse.jgit.revwalk.RevCommit;
|
import org.eclipse.jgit.revwalk.RevCommit;
|
||||||
import org.eclipse.jgit.revwalk.RevWalk;
|
import org.eclipse.jgit.revwalk.RevWalk;
|
||||||
import org.eclipse.jgit.storage.pack.PackConfig;
|
import org.eclipse.jgit.storage.pack.PackConfig;
|
||||||
|
import org.eclipse.jgit.transport.ReceiveCommand;
|
||||||
import org.eclipse.jgit.util.SystemReader;
|
import org.eclipse.jgit.util.SystemReader;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -653,6 +664,185 @@ public void testSinglePackForAllRefs() throws Exception {
|
||||||
assertEquals(2, odb.getPacks().length);
|
assertEquals(2, odb.getPacks().length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("boxing")
|
||||||
|
@Test
|
||||||
|
public void producesNewReftable() throws Exception {
|
||||||
|
String master = "refs/heads/master";
|
||||||
|
RevCommit commit0 = commit().message("0").create();
|
||||||
|
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||||
|
|
||||||
|
BatchRefUpdate bru = git.getRepository().getRefDatabase()
|
||||||
|
.newBatchUpdate();
|
||||||
|
bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
|
||||||
|
for (int i = 1; i <= 5100; i++) {
|
||||||
|
bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
|
||||||
|
String.format("refs/pulls/%04d", i)));
|
||||||
|
}
|
||||||
|
try (RevWalk rw = new RevWalk(git.getRepository())) {
|
||||||
|
bru.execute(rw, NullProgressMonitor.INSTANCE);
|
||||||
|
}
|
||||||
|
|
||||||
|
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||||
|
gc.setReftableConfig(new ReftableConfig());
|
||||||
|
run(gc);
|
||||||
|
|
||||||
|
// Single GC pack present with all objects.
|
||||||
|
assertEquals(1, odb.getPacks().length);
|
||||||
|
DfsPackFile pack = odb.getPacks()[0];
|
||||||
|
DfsPackDescription desc = pack.getPackDescription();
|
||||||
|
assertEquals(GC, desc.getPackSource());
|
||||||
|
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||||
|
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||||
|
|
||||||
|
// Sibling REFTABLE is also present.
|
||||||
|
assertTrue(desc.hasFileExt(REFTABLE));
|
||||||
|
ReftableWriter.Stats stats = desc.getReftableStats();
|
||||||
|
assertNotNull(stats);
|
||||||
|
assertTrue(stats.totalBytes() > 0);
|
||||||
|
assertEquals(5101, stats.refCount());
|
||||||
|
assertEquals(1, stats.minUpdateIndex());
|
||||||
|
assertEquals(1, stats.maxUpdateIndex());
|
||||||
|
|
||||||
|
DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
|
||||||
|
try (DfsReader ctx = odb.newReader();
|
||||||
|
ReftableReader rr = table.open(ctx);
|
||||||
|
RefCursor rc = rr.seekRef("refs/pulls/5100")) {
|
||||||
|
assertTrue(rc.next());
|
||||||
|
assertEquals(commit0, rc.getRef().getObjectId());
|
||||||
|
assertFalse(rc.next());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void leavesNonGcReftablesIfNotConfigured() throws Exception {
|
||||||
|
String master = "refs/heads/master";
|
||||||
|
RevCommit commit0 = commit().message("0").create();
|
||||||
|
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||||
|
git.update(master, commit1);
|
||||||
|
|
||||||
|
DfsPackDescription t1 = odb.newPack(INSERT);
|
||||||
|
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
|
||||||
|
out.write("ignored".getBytes(StandardCharsets.UTF_8));
|
||||||
|
t1.addFileExt(REFTABLE);
|
||||||
|
}
|
||||||
|
odb.commitPack(Collections.singleton(t1), null);
|
||||||
|
|
||||||
|
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||||
|
gc.setReftableConfig(null);
|
||||||
|
run(gc);
|
||||||
|
|
||||||
|
// Single GC pack present with all objects.
|
||||||
|
assertEquals(1, odb.getPacks().length);
|
||||||
|
DfsPackFile pack = odb.getPacks()[0];
|
||||||
|
DfsPackDescription desc = pack.getPackDescription();
|
||||||
|
assertEquals(GC, desc.getPackSource());
|
||||||
|
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||||
|
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||||
|
|
||||||
|
// Only INSERT REFTABLE above is present.
|
||||||
|
DfsReftable[] tables = odb.getReftables();
|
||||||
|
assertEquals(1, tables.length);
|
||||||
|
assertEquals(t1, tables[0].getPackDescription());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void prunesNonGcReftables() throws Exception {
|
||||||
|
String master = "refs/heads/master";
|
||||||
|
RevCommit commit0 = commit().message("0").create();
|
||||||
|
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||||
|
git.update(master, commit1);
|
||||||
|
|
||||||
|
DfsPackDescription t1 = odb.newPack(INSERT);
|
||||||
|
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
|
||||||
|
out.write("ignored".getBytes(StandardCharsets.UTF_8));
|
||||||
|
t1.addFileExt(REFTABLE);
|
||||||
|
}
|
||||||
|
odb.commitPack(Collections.singleton(t1), null);
|
||||||
|
odb.clearCache();
|
||||||
|
|
||||||
|
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||||
|
gc.setReftableConfig(new ReftableConfig());
|
||||||
|
run(gc);
|
||||||
|
|
||||||
|
// Single GC pack present with all objects.
|
||||||
|
assertEquals(1, odb.getPacks().length);
|
||||||
|
DfsPackFile pack = odb.getPacks()[0];
|
||||||
|
DfsPackDescription desc = pack.getPackDescription();
|
||||||
|
assertEquals(GC, desc.getPackSource());
|
||||||
|
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||||
|
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||||
|
|
||||||
|
// Only sibling GC REFTABLE is present.
|
||||||
|
DfsReftable[] tables = odb.getReftables();
|
||||||
|
assertEquals(1, tables.length);
|
||||||
|
assertEquals(desc, tables[0].getPackDescription());
|
||||||
|
assertTrue(desc.hasFileExt(REFTABLE));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void compactsReftables() throws Exception {
|
||||||
|
String master = "refs/heads/master";
|
||||||
|
RevCommit commit0 = commit().message("0").create();
|
||||||
|
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||||
|
git.update(master, commit1);
|
||||||
|
|
||||||
|
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||||
|
gc.setReftableConfig(new ReftableConfig());
|
||||||
|
run(gc);
|
||||||
|
|
||||||
|
DfsPackDescription t1 = odb.newPack(INSERT);
|
||||||
|
Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
|
||||||
|
"refs/heads/next", commit0.copy());
|
||||||
|
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
|
||||||
|
ReftableWriter w = new ReftableWriter();
|
||||||
|
w.setMinUpdateIndex(42);
|
||||||
|
w.setMaxUpdateIndex(42);
|
||||||
|
w.begin(out);
|
||||||
|
w.sortAndWriteRefs(Collections.singleton(next));
|
||||||
|
w.finish();
|
||||||
|
t1.addFileExt(REFTABLE);
|
||||||
|
t1.setReftableStats(w.getStats());
|
||||||
|
}
|
||||||
|
odb.commitPack(Collections.singleton(t1), null);
|
||||||
|
|
||||||
|
gc = new DfsGarbageCollector(repo);
|
||||||
|
gc.setReftableConfig(new ReftableConfig());
|
||||||
|
run(gc);
|
||||||
|
|
||||||
|
// Single GC pack present with all objects.
|
||||||
|
assertEquals(1, odb.getPacks().length);
|
||||||
|
DfsPackFile pack = odb.getPacks()[0];
|
||||||
|
DfsPackDescription desc = pack.getPackDescription();
|
||||||
|
assertEquals(GC, desc.getPackSource());
|
||||||
|
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||||
|
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||||
|
|
||||||
|
// Only sibling GC REFTABLE is present.
|
||||||
|
DfsReftable[] tables = odb.getReftables();
|
||||||
|
assertEquals(1, tables.length);
|
||||||
|
assertEquals(desc, tables[0].getPackDescription());
|
||||||
|
assertTrue(desc.hasFileExt(REFTABLE));
|
||||||
|
|
||||||
|
// GC reftable contains the compaction.
|
||||||
|
DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
|
||||||
|
try (DfsReader ctx = odb.newReader();
|
||||||
|
ReftableReader rr = table.open(ctx);
|
||||||
|
RefCursor rc = rr.allRefs()) {
|
||||||
|
assertEquals(1, rr.minUpdateIndex());
|
||||||
|
assertEquals(42, rr.maxUpdateIndex());
|
||||||
|
|
||||||
|
assertTrue(rc.next());
|
||||||
|
assertEquals(master, rc.getRef().getName());
|
||||||
|
assertEquals(commit1, rc.getRef().getObjectId());
|
||||||
|
|
||||||
|
assertTrue(rc.next());
|
||||||
|
assertEquals(next.getName(), rc.getRef().getName());
|
||||||
|
assertEquals(commit0, rc.getRef().getObjectId());
|
||||||
|
|
||||||
|
assertFalse(rc.next());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private TestRepository<InMemoryRepository>.CommitBuilder commit() {
|
private TestRepository<InMemoryRepository>.CommitBuilder commit() {
|
||||||
return git.commit();
|
return git.commit();
|
||||||
}
|
}
|
||||||
|
|
|
@ -50,13 +50,16 @@
|
||||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
|
||||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE;
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE;
|
||||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
|
||||||
|
import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
||||||
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
|
import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
@ -72,6 +75,9 @@
|
||||||
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
|
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackExt;
|
import org.eclipse.jgit.internal.storage.pack.PackExt;
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
|
||||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
|
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
@ -94,14 +100,15 @@ public class DfsGarbageCollector {
|
||||||
private final DfsObjDatabase objdb;
|
private final DfsObjDatabase objdb;
|
||||||
|
|
||||||
private final List<DfsPackDescription> newPackDesc;
|
private final List<DfsPackDescription> newPackDesc;
|
||||||
|
|
||||||
private final List<PackStatistics> newPackStats;
|
private final List<PackStatistics> newPackStats;
|
||||||
|
|
||||||
private final List<ObjectIdSet> newPackObj;
|
private final List<ObjectIdSet> newPackObj;
|
||||||
|
|
||||||
private DfsReader ctx;
|
private DfsReader ctx;
|
||||||
|
|
||||||
private PackConfig packConfig;
|
private PackConfig packConfig;
|
||||||
|
private ReftableConfig reftableConfig;
|
||||||
|
private long reftableInitialMinUpdateIndex = 1;
|
||||||
|
private long reftableInitialMaxUpdateIndex = 1;
|
||||||
|
|
||||||
// See packIsCoalesceableGarbage(), below, for how these two variables
|
// See packIsCoalesceableGarbage(), below, for how these two variables
|
||||||
// interact.
|
// interact.
|
||||||
|
@ -110,8 +117,10 @@ public class DfsGarbageCollector {
|
||||||
|
|
||||||
private long startTimeMillis;
|
private long startTimeMillis;
|
||||||
private List<DfsPackFile> packsBefore;
|
private List<DfsPackFile> packsBefore;
|
||||||
|
private List<DfsReftable> reftablesBefore;
|
||||||
private List<DfsPackFile> expiredGarbagePacks;
|
private List<DfsPackFile> expiredGarbagePacks;
|
||||||
|
|
||||||
|
private Collection<Ref> refsBefore;
|
||||||
private Set<ObjectId> allHeadsAndTags;
|
private Set<ObjectId> allHeadsAndTags;
|
||||||
private Set<ObjectId> allTags;
|
private Set<ObjectId> allTags;
|
||||||
private Set<ObjectId> nonHeads;
|
private Set<ObjectId> nonHeads;
|
||||||
|
@ -151,6 +160,57 @@ public DfsGarbageCollector setPackConfig(PackConfig newConfig) {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param cfg
|
||||||
|
* configuration to write a reftable. Reftable writing is
|
||||||
|
* disabled (default) when {@code cfg} is {@code null}.
|
||||||
|
* @return {@code this}
|
||||||
|
*/
|
||||||
|
public DfsGarbageCollector setReftableConfig(ReftableConfig cfg) {
|
||||||
|
reftableConfig = cfg;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set minUpdateIndex for the initial reftable created during conversion.
|
||||||
|
* <p>
|
||||||
|
* <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
|
||||||
|
* normally performed at the start of {@link #pack(ProgressMonitor)}.
|
||||||
|
* Callers must ensure the reference cache is current and will have been
|
||||||
|
* read before the pack list.
|
||||||
|
*
|
||||||
|
* @param u
|
||||||
|
* minUpdateIndex for the initial reftable created by scanning
|
||||||
|
* {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
|
||||||
|
* has also set {@link #setReftableConfig(ReftableConfig)}.
|
||||||
|
* Defaults to {@code 1}. Must be {@code u >= 0}.
|
||||||
|
* @return {@code this}
|
||||||
|
*/
|
||||||
|
public DfsGarbageCollector setReftableInitialMinUpdateIndex(long u) {
|
||||||
|
reftableInitialMinUpdateIndex = Math.max(u, 0);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set maxUpdateIndex for the initial reftable created during conversion.
|
||||||
|
* <p>
|
||||||
|
* <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
|
||||||
|
* normally performed at the start of {@link #pack(ProgressMonitor)}.
|
||||||
|
* Callers must ensure the reference cache is current and will have been
|
||||||
|
* read before the pack list.
|
||||||
|
*
|
||||||
|
* @param u
|
||||||
|
* maxUpdateIndex for the initial reftable created by scanning
|
||||||
|
* {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
|
||||||
|
* has also set {@link #setReftableConfig(ReftableConfig)}.
|
||||||
|
* Defaults to {@code 1}. Must be {@code u >= 0}.
|
||||||
|
* @return {@code this}
|
||||||
|
*/
|
||||||
|
public DfsGarbageCollector setReftableInitialMaxUpdateIndex(long u) {
|
||||||
|
reftableInitialMaxUpdateIndex = Math.max(0, u);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
/** @return garbage packs smaller than this size will be repacked. */
|
/** @return garbage packs smaller than this size will be repacked. */
|
||||||
public long getCoalesceGarbageLimit() {
|
public long getCoalesceGarbageLimit() {
|
||||||
return coalesceGarbageLimit;
|
return coalesceGarbageLimit;
|
||||||
|
@ -240,8 +300,9 @@ public boolean pack(ProgressMonitor pm) throws IOException {
|
||||||
refdb.refresh();
|
refdb.refresh();
|
||||||
objdb.clearCache();
|
objdb.clearCache();
|
||||||
|
|
||||||
Collection<Ref> refsBefore = getAllRefs();
|
refsBefore = getAllRefs();
|
||||||
readPacksBefore();
|
readPacksBefore();
|
||||||
|
readReftablesBefore();
|
||||||
|
|
||||||
Set<ObjectId> allHeads = new HashSet<>();
|
Set<ObjectId> allHeads = new HashSet<>();
|
||||||
allHeadsAndTags = new HashSet<>();
|
allHeadsAndTags = new HashSet<>();
|
||||||
|
@ -333,6 +394,11 @@ private void readPacksBefore() throws IOException {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void readReftablesBefore() throws IOException {
|
||||||
|
DfsReftable[] tables = objdb.getReftables();
|
||||||
|
reftablesBefore = new ArrayList<>(Arrays.asList(tables));
|
||||||
|
}
|
||||||
|
|
||||||
private boolean packIsExpiredGarbage(DfsPackDescription d, long now) {
|
private boolean packIsExpiredGarbage(DfsPackDescription d, long now) {
|
||||||
// Consider the garbage pack as expired when it's older than
|
// Consider the garbage pack as expired when it's older than
|
||||||
// garbagePackTtl. This check gives concurrent inserter threads
|
// garbagePackTtl. This check gives concurrent inserter threads
|
||||||
|
@ -407,7 +473,7 @@ private static long dayStartInMillis(long timeInMillis) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return all of the source packs that fed into this compaction. */
|
/** @return all of the source packs that fed into this compaction. */
|
||||||
public List<DfsPackDescription> getSourcePacks() {
|
public Set<DfsPackDescription> getSourcePacks() {
|
||||||
return toPrune();
|
return toPrune();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -421,28 +487,37 @@ public List<PackStatistics> getNewPackStatistics() {
|
||||||
return newPackStats;
|
return newPackStats;
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<DfsPackDescription> toPrune() {
|
private Set<DfsPackDescription> toPrune() {
|
||||||
int cnt = packsBefore.size();
|
Set<DfsPackDescription> toPrune = new HashSet<>();
|
||||||
List<DfsPackDescription> all = new ArrayList<>(cnt);
|
|
||||||
for (DfsPackFile pack : packsBefore) {
|
for (DfsPackFile pack : packsBefore) {
|
||||||
all.add(pack.getPackDescription());
|
toPrune.add(pack.getPackDescription());
|
||||||
|
}
|
||||||
|
if (reftableConfig != null) {
|
||||||
|
for (DfsReftable table : reftablesBefore) {
|
||||||
|
toPrune.add(table.getPackDescription());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for (DfsPackFile pack : expiredGarbagePacks) {
|
for (DfsPackFile pack : expiredGarbagePacks) {
|
||||||
all.add(pack.getPackDescription());
|
toPrune.add(pack.getPackDescription());
|
||||||
}
|
}
|
||||||
return all;
|
return toPrune;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void packHeads(ProgressMonitor pm) throws IOException {
|
private void packHeads(ProgressMonitor pm) throws IOException {
|
||||||
if (allHeadsAndTags.isEmpty())
|
if (allHeadsAndTags.isEmpty()) {
|
||||||
|
writeReftable();
|
||||||
return;
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try (PackWriter pw = newPackWriter()) {
|
try (PackWriter pw = newPackWriter()) {
|
||||||
pw.setTagTargets(tagTargets);
|
pw.setTagTargets(tagTargets);
|
||||||
pw.preparePack(pm, allHeadsAndTags, NONE, NONE, allTags);
|
pw.preparePack(pm, allHeadsAndTags, NONE, NONE, allTags);
|
||||||
if (0 < pw.getObjectCount())
|
if (0 < pw.getObjectCount()) {
|
||||||
writePack(GC, pw, pm,
|
long estSize = estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC);
|
||||||
estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC));
|
writePack(GC, pw, pm, estSize);
|
||||||
|
} else {
|
||||||
|
writeReftable();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -560,6 +635,10 @@ private DfsPackDescription writePack(PackSource source, PackWriter pw,
|
||||||
estimatedPackSize);
|
estimatedPackSize);
|
||||||
newPackDesc.add(pack);
|
newPackDesc.add(pack);
|
||||||
|
|
||||||
|
if (source == GC && reftableConfig != null) {
|
||||||
|
writeReftable(pack);
|
||||||
|
}
|
||||||
|
|
||||||
try (DfsOutputStream out = objdb.writeFile(pack, PACK)) {
|
try (DfsOutputStream out = objdb.writeFile(pack, PACK)) {
|
||||||
pw.writePack(pm, pm, out);
|
pw.writePack(pm, pm, out);
|
||||||
pack.addFileExt(PACK);
|
pack.addFileExt(PACK);
|
||||||
|
@ -592,4 +671,60 @@ private DfsPackDescription writePack(PackSource source, PackWriter pw,
|
||||||
newPackObj.add(pw.getObjectSet());
|
newPackObj.add(pw.getObjectSet());
|
||||||
return pack;
|
return pack;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void writeReftable() throws IOException {
|
||||||
|
if (reftableConfig != null) {
|
||||||
|
DfsPackDescription pack = objdb.newPack(GC);
|
||||||
|
newPackDesc.add(pack);
|
||||||
|
writeReftable(pack);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeReftable(DfsPackDescription pack) throws IOException {
|
||||||
|
if (!hasGcReftable()) {
|
||||||
|
writeReftable(pack, refsBefore);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try (ReftableStack stack = ReftableStack.open(ctx, reftablesBefore)) {
|
||||||
|
ReftableCompactor compact = new ReftableCompactor();
|
||||||
|
compact.addAll(stack.readers());
|
||||||
|
compact.setIncludeDeletes(false);
|
||||||
|
compactReftable(pack, compact);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasGcReftable() {
|
||||||
|
for (DfsReftable table : reftablesBefore) {
|
||||||
|
if (table.getPackDescription().getPackSource() == GC) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeReftable(DfsPackDescription pack, Collection<Ref> refs)
|
||||||
|
throws IOException {
|
||||||
|
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
|
||||||
|
ReftableConfig cfg = configureReftable(reftableConfig, out);
|
||||||
|
ReftableWriter writer = new ReftableWriter(cfg)
|
||||||
|
.setMinUpdateIndex(reftableInitialMinUpdateIndex)
|
||||||
|
.setMaxUpdateIndex(reftableInitialMaxUpdateIndex)
|
||||||
|
.begin(out)
|
||||||
|
.sortAndWriteRefs(refs)
|
||||||
|
.finish();
|
||||||
|
pack.addFileExt(REFTABLE);
|
||||||
|
pack.setReftableStats(writer.getStats());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void compactReftable(DfsPackDescription pack,
|
||||||
|
ReftableCompactor compact) throws IOException {
|
||||||
|
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
|
||||||
|
compact.setConfig(configureReftable(reftableConfig, out));
|
||||||
|
compact.compact(out);
|
||||||
|
pack.addFileExt(REFTABLE);
|
||||||
|
pack.setReftableStats(compact.getStats());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,21 +44,29 @@
|
||||||
package org.eclipse.jgit.internal.storage.dfs;
|
package org.eclipse.jgit.internal.storage.dfs;
|
||||||
|
|
||||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
|
||||||
|
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
||||||
|
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
|
||||||
import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
|
import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||||
import org.eclipse.jgit.internal.JGitText;
|
import org.eclipse.jgit.internal.JGitText;
|
||||||
import org.eclipse.jgit.internal.storage.file.PackIndex;
|
import org.eclipse.jgit.internal.storage.file.PackIndex;
|
||||||
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
|
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
|
||||||
|
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
import org.eclipse.jgit.lib.NullProgressMonitor;
|
import org.eclipse.jgit.lib.NullProgressMonitor;
|
||||||
import org.eclipse.jgit.lib.ObjectId;
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
|
@ -89,16 +97,15 @@
|
||||||
*/
|
*/
|
||||||
public class DfsPackCompactor {
|
public class DfsPackCompactor {
|
||||||
private final DfsRepository repo;
|
private final DfsRepository repo;
|
||||||
|
|
||||||
private final List<DfsPackFile> srcPacks;
|
private final List<DfsPackFile> srcPacks;
|
||||||
|
private final List<DfsReftable> srcReftables;
|
||||||
private final List<ObjectIdSet> exclude;
|
private final List<ObjectIdSet> exclude;
|
||||||
|
|
||||||
private final List<DfsPackDescription> newPacks;
|
private PackStatistics newStats;
|
||||||
|
private DfsPackDescription outDesc;
|
||||||
private final List<PackStatistics> newStats;
|
|
||||||
|
|
||||||
private int autoAddSize;
|
private int autoAddSize;
|
||||||
|
private ReftableConfig reftableConfig;
|
||||||
|
|
||||||
private RevWalk rw;
|
private RevWalk rw;
|
||||||
private RevFlag added;
|
private RevFlag added;
|
||||||
|
@ -114,9 +121,19 @@ public DfsPackCompactor(DfsRepository repository) {
|
||||||
repo = repository;
|
repo = repository;
|
||||||
autoAddSize = 5 * 1024 * 1024; // 5 MiB
|
autoAddSize = 5 * 1024 * 1024; // 5 MiB
|
||||||
srcPacks = new ArrayList<>();
|
srcPacks = new ArrayList<>();
|
||||||
|
srcReftables = new ArrayList<>();
|
||||||
exclude = new ArrayList<>(4);
|
exclude = new ArrayList<>(4);
|
||||||
newPacks = new ArrayList<>(1);
|
}
|
||||||
newStats = new ArrayList<>(1);
|
|
||||||
|
/**
|
||||||
|
* @param cfg
|
||||||
|
* configuration to write a reftable. Reftable compacting is
|
||||||
|
* disabled (default) when {@code cfg} is {@code null}.
|
||||||
|
* @return {@code this}
|
||||||
|
*/
|
||||||
|
public DfsPackCompactor setReftableConfig(ReftableConfig cfg) {
|
||||||
|
reftableConfig = cfg;
|
||||||
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -137,7 +154,19 @@ public DfsPackCompactor add(DfsPackFile pack) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Automatically select packs to be included, and add them.
|
* Add a reftable to be compacted.
|
||||||
|
*
|
||||||
|
* @param table
|
||||||
|
* a reftable to combine.
|
||||||
|
* @return {@code this}
|
||||||
|
*/
|
||||||
|
public DfsPackCompactor add(DfsReftable table) {
|
||||||
|
srcReftables.add(table);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Automatically select pack and reftables to be included, and add them.
|
||||||
* <p>
|
* <p>
|
||||||
* Packs are selected based on size, smaller packs get included while bigger
|
* Packs are selected based on size, smaller packs get included while bigger
|
||||||
* ones are omitted.
|
* ones are omitted.
|
||||||
|
@ -155,6 +184,16 @@ public DfsPackCompactor autoAdd() throws IOException {
|
||||||
else
|
else
|
||||||
exclude(pack);
|
exclude(pack);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (reftableConfig != null) {
|
||||||
|
for (DfsReftable table : objdb.getReftables()) {
|
||||||
|
DfsPackDescription d = table.getPackDescription();
|
||||||
|
if (d.getPackSource() != GC
|
||||||
|
&& d.getFileSize(REFTABLE) < autoAddSize) {
|
||||||
|
add(table);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -197,11 +236,30 @@ public DfsPackCompactor exclude(DfsPackFile pack) throws IOException {
|
||||||
* the packs cannot be compacted.
|
* the packs cannot be compacted.
|
||||||
*/
|
*/
|
||||||
public void compact(ProgressMonitor pm) throws IOException {
|
public void compact(ProgressMonitor pm) throws IOException {
|
||||||
if (pm == null)
|
if (pm == null) {
|
||||||
pm = NullProgressMonitor.INSTANCE;
|
pm = NullProgressMonitor.INSTANCE;
|
||||||
|
}
|
||||||
|
|
||||||
DfsObjDatabase objdb = repo.getObjectDatabase();
|
DfsObjDatabase objdb = repo.getObjectDatabase();
|
||||||
try (DfsReader ctx = objdb.newReader()) {
|
try (DfsReader ctx = objdb.newReader()) {
|
||||||
|
if (reftableConfig != null && !srcReftables.isEmpty()) {
|
||||||
|
compactReftables(ctx);
|
||||||
|
}
|
||||||
|
compactPacks(ctx, pm);
|
||||||
|
|
||||||
|
List<DfsPackDescription> commit = getNewPacks();
|
||||||
|
Collection<DfsPackDescription> remove = toPrune();
|
||||||
|
if (!commit.isEmpty() || !remove.isEmpty()) {
|
||||||
|
objdb.commitPack(commit, remove);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
rw = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void compactPacks(DfsReader ctx, ProgressMonitor pm)
|
||||||
|
throws IOException, IncorrectObjectTypeException {
|
||||||
|
DfsObjDatabase objdb = repo.getObjectDatabase();
|
||||||
PackConfig pc = new PackConfig(repo);
|
PackConfig pc = new PackConfig(repo);
|
||||||
pc.setIndexVersion(2);
|
pc.setIndexVersion(2);
|
||||||
pc.setDeltaCompress(false);
|
pc.setDeltaCompress(false);
|
||||||
|
@ -215,40 +273,31 @@ public void compact(ProgressMonitor pm) throws IOException {
|
||||||
|
|
||||||
addObjectsToPack(pw, ctx, pm);
|
addObjectsToPack(pw, ctx, pm);
|
||||||
if (pw.getObjectCount() == 0) {
|
if (pw.getObjectCount() == 0) {
|
||||||
List<DfsPackDescription> remove = toPrune();
|
|
||||||
if (remove.size() > 0)
|
|
||||||
objdb.commitPack(
|
|
||||||
Collections.<DfsPackDescription>emptyList(),
|
|
||||||
remove);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean rollback = true;
|
boolean rollback = true;
|
||||||
DfsPackDescription pack = objdb.newPack(COMPACT,
|
initOutDesc(objdb);
|
||||||
estimatePackSize());
|
|
||||||
try {
|
try {
|
||||||
writePack(objdb, pack, pw, pm);
|
writePack(objdb, outDesc, pw, pm);
|
||||||
writeIndex(objdb, pack, pw);
|
writeIndex(objdb, outDesc, pw);
|
||||||
|
|
||||||
PackStatistics stats = pw.getStatistics();
|
PackStatistics stats = pw.getStatistics();
|
||||||
pw.close();
|
pw.close();
|
||||||
pw = null;
|
pw = null;
|
||||||
|
|
||||||
pack.setPackStats(stats);
|
outDesc.setPackStats(stats);
|
||||||
objdb.commitPack(Collections.singletonList(pack), toPrune());
|
newStats = stats;
|
||||||
newPacks.add(pack);
|
|
||||||
newStats.add(stats);
|
|
||||||
rollback = false;
|
rollback = false;
|
||||||
} finally {
|
} finally {
|
||||||
if (rollback)
|
if (rollback) {
|
||||||
objdb.rollbackPack(Collections.singletonList(pack));
|
objdb.rollbackPack(Collections.singletonList(outDesc));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (pw != null)
|
if (pw != null) {
|
||||||
pw.close();
|
pw.close();
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
rw = null;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -263,27 +312,81 @@ private long estimatePackSize() {
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void compactReftables(DfsReader ctx) throws IOException {
|
||||||
|
DfsObjDatabase objdb = repo.getObjectDatabase();
|
||||||
|
Collections.sort(srcReftables, objdb.reftableComparator());
|
||||||
|
|
||||||
|
try (ReftableStack stack = ReftableStack.open(ctx, srcReftables)) {
|
||||||
|
initOutDesc(objdb);
|
||||||
|
ReftableCompactor compact = new ReftableCompactor();
|
||||||
|
compact.addAll(stack.readers());
|
||||||
|
compact.setIncludeDeletes(true);
|
||||||
|
writeReftable(objdb, outDesc, compact);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initOutDesc(DfsObjDatabase objdb) throws IOException {
|
||||||
|
if (outDesc == null) {
|
||||||
|
outDesc = objdb.newPack(COMPACT, estimatePackSize());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** @return all of the source packs that fed into this compaction. */
|
/** @return all of the source packs that fed into this compaction. */
|
||||||
public List<DfsPackDescription> getSourcePacks() {
|
public Collection<DfsPackDescription> getSourcePacks() {
|
||||||
return toPrune();
|
Set<DfsPackDescription> src = new HashSet<>();
|
||||||
|
for (DfsPackFile pack : srcPacks) {
|
||||||
|
src.add(pack.getPackDescription());
|
||||||
|
}
|
||||||
|
for (DfsReftable table : srcReftables) {
|
||||||
|
src.add(table.getPackDescription());
|
||||||
|
}
|
||||||
|
return src;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return new packs created by this compaction. */
|
/** @return new packs created by this compaction. */
|
||||||
public List<DfsPackDescription> getNewPacks() {
|
public List<DfsPackDescription> getNewPacks() {
|
||||||
return newPacks;
|
return outDesc != null
|
||||||
|
? Collections.singletonList(outDesc)
|
||||||
|
: Collections.emptyList();
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return statistics corresponding to the {@link #getNewPacks()}. */
|
/** @return statistics corresponding to the {@link #getNewPacks()}. */
|
||||||
public List<PackStatistics> getNewPackStatistics() {
|
public List<PackStatistics> getNewPackStatistics() {
|
||||||
return newStats;
|
return newStats != null
|
||||||
|
? Collections.singletonList(newStats)
|
||||||
|
: Collections.emptyList();
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<DfsPackDescription> toPrune() {
|
private Collection<DfsPackDescription> toPrune() {
|
||||||
int cnt = srcPacks.size();
|
Set<DfsPackDescription> packs = new HashSet<>();
|
||||||
List<DfsPackDescription> all = new ArrayList<>(cnt);
|
for (DfsPackFile pack : srcPacks) {
|
||||||
for (DfsPackFile pack : srcPacks)
|
packs.add(pack.getPackDescription());
|
||||||
all.add(pack.getPackDescription());
|
}
|
||||||
return all;
|
|
||||||
|
Set<DfsPackDescription> reftables = new HashSet<>();
|
||||||
|
for (DfsReftable table : srcReftables) {
|
||||||
|
reftables.add(table.getPackDescription());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Iterator<DfsPackDescription> i = packs.iterator(); i.hasNext();) {
|
||||||
|
DfsPackDescription d = i.next();
|
||||||
|
if (d.hasFileExt(REFTABLE) && !reftables.contains(d)) {
|
||||||
|
i.remove();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Iterator<DfsPackDescription> i = reftables.iterator();
|
||||||
|
i.hasNext();) {
|
||||||
|
DfsPackDescription d = i.next();
|
||||||
|
if (d.hasFileExt(PACK) && !packs.contains(d)) {
|
||||||
|
i.remove();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Set<DfsPackDescription> toPrune = new HashSet<>();
|
||||||
|
toPrune.addAll(packs);
|
||||||
|
toPrune.addAll(reftables);
|
||||||
|
return toPrune;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addObjectsToPack(PackWriter pw, DfsReader ctx,
|
private void addObjectsToPack(PackWriter pw, DfsReader ctx,
|
||||||
|
@ -390,6 +493,27 @@ private static void writeIndex(DfsObjDatabase objdb,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void writeReftable(DfsObjDatabase objdb, DfsPackDescription pack,
|
||||||
|
ReftableCompactor compact) throws IOException {
|
||||||
|
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
|
||||||
|
compact.setConfig(configureReftable(reftableConfig, out));
|
||||||
|
compact.compact(out);
|
||||||
|
pack.addFileExt(REFTABLE);
|
||||||
|
pack.setReftableStats(compact.getStats());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReftableConfig configureReftable(ReftableConfig cfg,
|
||||||
|
DfsOutputStream out) {
|
||||||
|
int bs = out.blockSize();
|
||||||
|
if (bs > 0) {
|
||||||
|
cfg = new ReftableConfig(cfg);
|
||||||
|
cfg.setRefBlockSize(bs);
|
||||||
|
cfg.setAlignBlocks(true);
|
||||||
|
}
|
||||||
|
return cfg;
|
||||||
|
}
|
||||||
|
|
||||||
private static class ObjectIdWithOffset extends ObjectId {
|
private static class ObjectIdWithOffset extends ObjectId {
|
||||||
final long offset;
|
final long offset;
|
||||||
|
|
||||||
|
|
|
@ -159,9 +159,16 @@ public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) {
|
||||||
* tables to compact. Tables should be ordered oldest first/most
|
* tables to compact. Tables should be ordered oldest first/most
|
||||||
* recent last so that the more recent tables can shadow the
|
* recent last so that the more recent tables can shadow the
|
||||||
* older results. Caller is responsible for closing the readers.
|
* older results. Caller is responsible for closing the readers.
|
||||||
|
* @throws IOException
|
||||||
|
* update indexes of a reader cannot be accessed.
|
||||||
*/
|
*/
|
||||||
public void addAll(List<? extends Reftable> readers) {
|
public void addAll(List<? extends Reftable> readers) throws IOException {
|
||||||
tables.addAll(readers);
|
tables.addAll(readers);
|
||||||
|
for (Reftable r : readers) {
|
||||||
|
if (r instanceof ReftableReader) {
|
||||||
|
adjustUpdateIndexes((ReftableReader) r);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -178,7 +185,7 @@ public void addAll(List<? extends Reftable> readers) {
|
||||||
* @return {@code true} if the compactor accepted this table; {@code false}
|
* @return {@code true} if the compactor accepted this table; {@code false}
|
||||||
* if the compactor has reached its limit.
|
* if the compactor has reached its limit.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* if size of {@code reader} cannot be read.
|
* if size of {@code reader}, or its update indexes cannot be read.
|
||||||
*/
|
*/
|
||||||
public boolean tryAddFirst(ReftableReader reader) throws IOException {
|
public boolean tryAddFirst(ReftableReader reader) throws IOException {
|
||||||
long sz = reader.size();
|
long sz = reader.size();
|
||||||
|
@ -186,10 +193,20 @@ public boolean tryAddFirst(ReftableReader reader) throws IOException {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bytesToCompact += sz;
|
bytesToCompact += sz;
|
||||||
|
adjustUpdateIndexes(reader);
|
||||||
tables.addFirst(reader);
|
tables.addFirst(reader);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void adjustUpdateIndexes(ReftableReader reader) throws IOException {
|
||||||
|
if (minUpdateIndex == 0) {
|
||||||
|
minUpdateIndex = reader.minUpdateIndex();
|
||||||
|
} else {
|
||||||
|
minUpdateIndex = Math.min(minUpdateIndex, reader.minUpdateIndex());
|
||||||
|
}
|
||||||
|
maxUpdateIndex = Math.max(maxUpdateIndex, reader.maxUpdateIndex());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write a compaction to {@code out}.
|
* Write a compaction to {@code out}.
|
||||||
*
|
*
|
||||||
|
|
Loading…
Reference in New Issue