Merge changes from topic 'dfs-reftable'
* changes: dfs: write reftable from DfsGarbageCollector dfs: compact reftables during DfsPackCompactor
This commit is contained in:
commit
d26309c4d9
|
@ -5,6 +5,7 @@
|
|||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
|
||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
@ -13,19 +14,29 @@
|
|||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Collections;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
|
||||
import org.eclipse.jgit.internal.storage.reftable.RefCursor;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
|
||||
import org.eclipse.jgit.junit.MockSystemReader;
|
||||
import org.eclipse.jgit.junit.TestRepository;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.BatchRefUpdate;
|
||||
import org.eclipse.jgit.lib.NullProgressMonitor;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.ObjectIdRef;
|
||||
import org.eclipse.jgit.lib.Ref;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.revwalk.RevBlob;
|
||||
import org.eclipse.jgit.revwalk.RevCommit;
|
||||
import org.eclipse.jgit.revwalk.RevWalk;
|
||||
import org.eclipse.jgit.storage.pack.PackConfig;
|
||||
import org.eclipse.jgit.transport.ReceiveCommand;
|
||||
import org.eclipse.jgit.util.SystemReader;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
@ -653,6 +664,185 @@ public void testSinglePackForAllRefs() throws Exception {
|
|||
assertEquals(2, odb.getPacks().length);
|
||||
}
|
||||
|
||||
@SuppressWarnings("boxing")
|
||||
@Test
|
||||
public void producesNewReftable() throws Exception {
|
||||
String master = "refs/heads/master";
|
||||
RevCommit commit0 = commit().message("0").create();
|
||||
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||
|
||||
BatchRefUpdate bru = git.getRepository().getRefDatabase()
|
||||
.newBatchUpdate();
|
||||
bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
|
||||
for (int i = 1; i <= 5100; i++) {
|
||||
bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
|
||||
String.format("refs/pulls/%04d", i)));
|
||||
}
|
||||
try (RevWalk rw = new RevWalk(git.getRepository())) {
|
||||
bru.execute(rw, NullProgressMonitor.INSTANCE);
|
||||
}
|
||||
|
||||
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||
gc.setReftableConfig(new ReftableConfig());
|
||||
run(gc);
|
||||
|
||||
// Single GC pack present with all objects.
|
||||
assertEquals(1, odb.getPacks().length);
|
||||
DfsPackFile pack = odb.getPacks()[0];
|
||||
DfsPackDescription desc = pack.getPackDescription();
|
||||
assertEquals(GC, desc.getPackSource());
|
||||
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||
|
||||
// Sibling REFTABLE is also present.
|
||||
assertTrue(desc.hasFileExt(REFTABLE));
|
||||
ReftableWriter.Stats stats = desc.getReftableStats();
|
||||
assertNotNull(stats);
|
||||
assertTrue(stats.totalBytes() > 0);
|
||||
assertEquals(5101, stats.refCount());
|
||||
assertEquals(1, stats.minUpdateIndex());
|
||||
assertEquals(1, stats.maxUpdateIndex());
|
||||
|
||||
DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
|
||||
try (DfsReader ctx = odb.newReader();
|
||||
ReftableReader rr = table.open(ctx);
|
||||
RefCursor rc = rr.seekRef("refs/pulls/5100")) {
|
||||
assertTrue(rc.next());
|
||||
assertEquals(commit0, rc.getRef().getObjectId());
|
||||
assertFalse(rc.next());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void leavesNonGcReftablesIfNotConfigured() throws Exception {
|
||||
String master = "refs/heads/master";
|
||||
RevCommit commit0 = commit().message("0").create();
|
||||
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||
git.update(master, commit1);
|
||||
|
||||
DfsPackDescription t1 = odb.newPack(INSERT);
|
||||
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
|
||||
out.write("ignored".getBytes(StandardCharsets.UTF_8));
|
||||
t1.addFileExt(REFTABLE);
|
||||
}
|
||||
odb.commitPack(Collections.singleton(t1), null);
|
||||
|
||||
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||
gc.setReftableConfig(null);
|
||||
run(gc);
|
||||
|
||||
// Single GC pack present with all objects.
|
||||
assertEquals(1, odb.getPacks().length);
|
||||
DfsPackFile pack = odb.getPacks()[0];
|
||||
DfsPackDescription desc = pack.getPackDescription();
|
||||
assertEquals(GC, desc.getPackSource());
|
||||
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||
|
||||
// Only INSERT REFTABLE above is present.
|
||||
DfsReftable[] tables = odb.getReftables();
|
||||
assertEquals(1, tables.length);
|
||||
assertEquals(t1, tables[0].getPackDescription());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void prunesNonGcReftables() throws Exception {
|
||||
String master = "refs/heads/master";
|
||||
RevCommit commit0 = commit().message("0").create();
|
||||
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||
git.update(master, commit1);
|
||||
|
||||
DfsPackDescription t1 = odb.newPack(INSERT);
|
||||
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
|
||||
out.write("ignored".getBytes(StandardCharsets.UTF_8));
|
||||
t1.addFileExt(REFTABLE);
|
||||
}
|
||||
odb.commitPack(Collections.singleton(t1), null);
|
||||
odb.clearCache();
|
||||
|
||||
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||
gc.setReftableConfig(new ReftableConfig());
|
||||
run(gc);
|
||||
|
||||
// Single GC pack present with all objects.
|
||||
assertEquals(1, odb.getPacks().length);
|
||||
DfsPackFile pack = odb.getPacks()[0];
|
||||
DfsPackDescription desc = pack.getPackDescription();
|
||||
assertEquals(GC, desc.getPackSource());
|
||||
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||
|
||||
// Only sibling GC REFTABLE is present.
|
||||
DfsReftable[] tables = odb.getReftables();
|
||||
assertEquals(1, tables.length);
|
||||
assertEquals(desc, tables[0].getPackDescription());
|
||||
assertTrue(desc.hasFileExt(REFTABLE));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void compactsReftables() throws Exception {
|
||||
String master = "refs/heads/master";
|
||||
RevCommit commit0 = commit().message("0").create();
|
||||
RevCommit commit1 = commit().message("1").parent(commit0).create();
|
||||
git.update(master, commit1);
|
||||
|
||||
DfsGarbageCollector gc = new DfsGarbageCollector(repo);
|
||||
gc.setReftableConfig(new ReftableConfig());
|
||||
run(gc);
|
||||
|
||||
DfsPackDescription t1 = odb.newPack(INSERT);
|
||||
Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
|
||||
"refs/heads/next", commit0.copy());
|
||||
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
|
||||
ReftableWriter w = new ReftableWriter();
|
||||
w.setMinUpdateIndex(42);
|
||||
w.setMaxUpdateIndex(42);
|
||||
w.begin(out);
|
||||
w.sortAndWriteRefs(Collections.singleton(next));
|
||||
w.finish();
|
||||
t1.addFileExt(REFTABLE);
|
||||
t1.setReftableStats(w.getStats());
|
||||
}
|
||||
odb.commitPack(Collections.singleton(t1), null);
|
||||
|
||||
gc = new DfsGarbageCollector(repo);
|
||||
gc.setReftableConfig(new ReftableConfig());
|
||||
run(gc);
|
||||
|
||||
// Single GC pack present with all objects.
|
||||
assertEquals(1, odb.getPacks().length);
|
||||
DfsPackFile pack = odb.getPacks()[0];
|
||||
DfsPackDescription desc = pack.getPackDescription();
|
||||
assertEquals(GC, desc.getPackSource());
|
||||
assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
|
||||
assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
|
||||
|
||||
// Only sibling GC REFTABLE is present.
|
||||
DfsReftable[] tables = odb.getReftables();
|
||||
assertEquals(1, tables.length);
|
||||
assertEquals(desc, tables[0].getPackDescription());
|
||||
assertTrue(desc.hasFileExt(REFTABLE));
|
||||
|
||||
// GC reftable contains the compaction.
|
||||
DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
|
||||
try (DfsReader ctx = odb.newReader();
|
||||
ReftableReader rr = table.open(ctx);
|
||||
RefCursor rc = rr.allRefs()) {
|
||||
assertEquals(1, rr.minUpdateIndex());
|
||||
assertEquals(42, rr.maxUpdateIndex());
|
||||
|
||||
assertTrue(rc.next());
|
||||
assertEquals(master, rc.getRef().getName());
|
||||
assertEquals(commit1, rc.getRef().getObjectId());
|
||||
|
||||
assertTrue(rc.next());
|
||||
assertEquals(next.getName(), rc.getRef().getName());
|
||||
assertEquals(commit0, rc.getRef().getObjectId());
|
||||
|
||||
assertFalse(rc.next());
|
||||
}
|
||||
}
|
||||
|
||||
private TestRepository<InMemoryRepository>.CommitBuilder commit() {
|
||||
return git.commit();
|
||||
}
|
||||
|
|
|
@ -50,13 +50,16 @@
|
|||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
|
||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE;
|
||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
|
||||
import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collection;
|
||||
import java.util.EnumSet;
|
||||
|
@ -72,6 +75,9 @@
|
|||
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
|
||||
import org.eclipse.jgit.internal.storage.pack.PackExt;
|
||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
|
||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
|
@ -94,14 +100,15 @@ public class DfsGarbageCollector {
|
|||
private final DfsObjDatabase objdb;
|
||||
|
||||
private final List<DfsPackDescription> newPackDesc;
|
||||
|
||||
private final List<PackStatistics> newPackStats;
|
||||
|
||||
private final List<ObjectIdSet> newPackObj;
|
||||
|
||||
private DfsReader ctx;
|
||||
|
||||
private PackConfig packConfig;
|
||||
private ReftableConfig reftableConfig;
|
||||
private long reftableInitialMinUpdateIndex = 1;
|
||||
private long reftableInitialMaxUpdateIndex = 1;
|
||||
|
||||
// See packIsCoalesceableGarbage(), below, for how these two variables
|
||||
// interact.
|
||||
|
@ -110,8 +117,10 @@ public class DfsGarbageCollector {
|
|||
|
||||
private long startTimeMillis;
|
||||
private List<DfsPackFile> packsBefore;
|
||||
private List<DfsReftable> reftablesBefore;
|
||||
private List<DfsPackFile> expiredGarbagePacks;
|
||||
|
||||
private Collection<Ref> refsBefore;
|
||||
private Set<ObjectId> allHeadsAndTags;
|
||||
private Set<ObjectId> allTags;
|
||||
private Set<ObjectId> nonHeads;
|
||||
|
@ -151,6 +160,57 @@ public DfsGarbageCollector setPackConfig(PackConfig newConfig) {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param cfg
|
||||
* configuration to write a reftable. Reftable writing is
|
||||
* disabled (default) when {@code cfg} is {@code null}.
|
||||
* @return {@code this}
|
||||
*/
|
||||
public DfsGarbageCollector setReftableConfig(ReftableConfig cfg) {
|
||||
reftableConfig = cfg;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set minUpdateIndex for the initial reftable created during conversion.
|
||||
* <p>
|
||||
* <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
|
||||
* normally performed at the start of {@link #pack(ProgressMonitor)}.
|
||||
* Callers must ensure the reference cache is current and will have been
|
||||
* read before the pack list.
|
||||
*
|
||||
* @param u
|
||||
* minUpdateIndex for the initial reftable created by scanning
|
||||
* {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
|
||||
* has also set {@link #setReftableConfig(ReftableConfig)}.
|
||||
* Defaults to {@code 1}. Must be {@code u >= 0}.
|
||||
* @return {@code this}
|
||||
*/
|
||||
public DfsGarbageCollector setReftableInitialMinUpdateIndex(long u) {
|
||||
reftableInitialMinUpdateIndex = Math.max(u, 0);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set maxUpdateIndex for the initial reftable created during conversion.
|
||||
* <p>
|
||||
* <b>Warning:</b> A setting {@code != 1} <b>disables cache refreshes</b>
|
||||
* normally performed at the start of {@link #pack(ProgressMonitor)}.
|
||||
* Callers must ensure the reference cache is current and will have been
|
||||
* read before the pack list.
|
||||
*
|
||||
* @param u
|
||||
* maxUpdateIndex for the initial reftable created by scanning
|
||||
* {@link DfsRefDatabase#getRefs(String)}. Ignored unless caller
|
||||
* has also set {@link #setReftableConfig(ReftableConfig)}.
|
||||
* Defaults to {@code 1}. Must be {@code u >= 0}.
|
||||
* @return {@code this}
|
||||
*/
|
||||
public DfsGarbageCollector setReftableInitialMaxUpdateIndex(long u) {
|
||||
reftableInitialMaxUpdateIndex = Math.max(0, u);
|
||||
return this;
|
||||
}
|
||||
|
||||
/** @return garbage packs smaller than this size will be repacked. */
|
||||
public long getCoalesceGarbageLimit() {
|
||||
return coalesceGarbageLimit;
|
||||
|
@ -240,8 +300,9 @@ public boolean pack(ProgressMonitor pm) throws IOException {
|
|||
refdb.refresh();
|
||||
objdb.clearCache();
|
||||
|
||||
Collection<Ref> refsBefore = getAllRefs();
|
||||
refsBefore = getAllRefs();
|
||||
readPacksBefore();
|
||||
readReftablesBefore();
|
||||
|
||||
Set<ObjectId> allHeads = new HashSet<>();
|
||||
allHeadsAndTags = new HashSet<>();
|
||||
|
@ -333,6 +394,11 @@ private void readPacksBefore() throws IOException {
|
|||
}
|
||||
}
|
||||
|
||||
private void readReftablesBefore() throws IOException {
|
||||
DfsReftable[] tables = objdb.getReftables();
|
||||
reftablesBefore = new ArrayList<>(Arrays.asList(tables));
|
||||
}
|
||||
|
||||
private boolean packIsExpiredGarbage(DfsPackDescription d, long now) {
|
||||
// Consider the garbage pack as expired when it's older than
|
||||
// garbagePackTtl. This check gives concurrent inserter threads
|
||||
|
@ -407,7 +473,7 @@ private static long dayStartInMillis(long timeInMillis) {
|
|||
}
|
||||
|
||||
/** @return all of the source packs that fed into this compaction. */
|
||||
public List<DfsPackDescription> getSourcePacks() {
|
||||
public Set<DfsPackDescription> getSourcePacks() {
|
||||
return toPrune();
|
||||
}
|
||||
|
||||
|
@ -421,28 +487,37 @@ public List<PackStatistics> getNewPackStatistics() {
|
|||
return newPackStats;
|
||||
}
|
||||
|
||||
private List<DfsPackDescription> toPrune() {
|
||||
int cnt = packsBefore.size();
|
||||
List<DfsPackDescription> all = new ArrayList<>(cnt);
|
||||
private Set<DfsPackDescription> toPrune() {
|
||||
Set<DfsPackDescription> toPrune = new HashSet<>();
|
||||
for (DfsPackFile pack : packsBefore) {
|
||||
all.add(pack.getPackDescription());
|
||||
toPrune.add(pack.getPackDescription());
|
||||
}
|
||||
if (reftableConfig != null) {
|
||||
for (DfsReftable table : reftablesBefore) {
|
||||
toPrune.add(table.getPackDescription());
|
||||
}
|
||||
}
|
||||
for (DfsPackFile pack : expiredGarbagePacks) {
|
||||
all.add(pack.getPackDescription());
|
||||
toPrune.add(pack.getPackDescription());
|
||||
}
|
||||
return all;
|
||||
return toPrune;
|
||||
}
|
||||
|
||||
private void packHeads(ProgressMonitor pm) throws IOException {
|
||||
if (allHeadsAndTags.isEmpty())
|
||||
if (allHeadsAndTags.isEmpty()) {
|
||||
writeReftable();
|
||||
return;
|
||||
}
|
||||
|
||||
try (PackWriter pw = newPackWriter()) {
|
||||
pw.setTagTargets(tagTargets);
|
||||
pw.preparePack(pm, allHeadsAndTags, NONE, NONE, allTags);
|
||||
if (0 < pw.getObjectCount())
|
||||
writePack(GC, pw, pm,
|
||||
estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC));
|
||||
if (0 < pw.getObjectCount()) {
|
||||
long estSize = estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC);
|
||||
writePack(GC, pw, pm, estSize);
|
||||
} else {
|
||||
writeReftable();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -560,6 +635,10 @@ private DfsPackDescription writePack(PackSource source, PackWriter pw,
|
|||
estimatedPackSize);
|
||||
newPackDesc.add(pack);
|
||||
|
||||
if (source == GC && reftableConfig != null) {
|
||||
writeReftable(pack);
|
||||
}
|
||||
|
||||
try (DfsOutputStream out = objdb.writeFile(pack, PACK)) {
|
||||
pw.writePack(pm, pm, out);
|
||||
pack.addFileExt(PACK);
|
||||
|
@ -592,4 +671,60 @@ private DfsPackDescription writePack(PackSource source, PackWriter pw,
|
|||
newPackObj.add(pw.getObjectSet());
|
||||
return pack;
|
||||
}
|
||||
|
||||
private void writeReftable() throws IOException {
|
||||
if (reftableConfig != null) {
|
||||
DfsPackDescription pack = objdb.newPack(GC);
|
||||
newPackDesc.add(pack);
|
||||
writeReftable(pack);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeReftable(DfsPackDescription pack) throws IOException {
|
||||
if (!hasGcReftable()) {
|
||||
writeReftable(pack, refsBefore);
|
||||
return;
|
||||
}
|
||||
|
||||
try (ReftableStack stack = ReftableStack.open(ctx, reftablesBefore)) {
|
||||
ReftableCompactor compact = new ReftableCompactor();
|
||||
compact.addAll(stack.readers());
|
||||
compact.setIncludeDeletes(false);
|
||||
compactReftable(pack, compact);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasGcReftable() {
|
||||
for (DfsReftable table : reftablesBefore) {
|
||||
if (table.getPackDescription().getPackSource() == GC) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void writeReftable(DfsPackDescription pack, Collection<Ref> refs)
|
||||
throws IOException {
|
||||
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
|
||||
ReftableConfig cfg = configureReftable(reftableConfig, out);
|
||||
ReftableWriter writer = new ReftableWriter(cfg)
|
||||
.setMinUpdateIndex(reftableInitialMinUpdateIndex)
|
||||
.setMaxUpdateIndex(reftableInitialMaxUpdateIndex)
|
||||
.begin(out)
|
||||
.sortAndWriteRefs(refs)
|
||||
.finish();
|
||||
pack.addFileExt(REFTABLE);
|
||||
pack.setReftableStats(writer.getStats());
|
||||
}
|
||||
}
|
||||
|
||||
private void compactReftable(DfsPackDescription pack,
|
||||
ReftableCompactor compact) throws IOException {
|
||||
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
|
||||
compact.setConfig(configureReftable(reftableConfig, out));
|
||||
compact.compact(out);
|
||||
pack.addFileExt(REFTABLE);
|
||||
pack.setReftableStats(compact.getStats());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -44,21 +44,29 @@
|
|||
package org.eclipse.jgit.internal.storage.dfs;
|
||||
|
||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
|
||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
|
||||
import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
|
||||
import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||
import org.eclipse.jgit.internal.JGitText;
|
||||
import org.eclipse.jgit.internal.storage.file.PackIndex;
|
||||
import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
|
||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
|
||||
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.NullProgressMonitor;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
|
@ -89,16 +97,15 @@
|
|||
*/
|
||||
public class DfsPackCompactor {
|
||||
private final DfsRepository repo;
|
||||
|
||||
private final List<DfsPackFile> srcPacks;
|
||||
|
||||
private final List<DfsReftable> srcReftables;
|
||||
private final List<ObjectIdSet> exclude;
|
||||
|
||||
private final List<DfsPackDescription> newPacks;
|
||||
|
||||
private final List<PackStatistics> newStats;
|
||||
private PackStatistics newStats;
|
||||
private DfsPackDescription outDesc;
|
||||
|
||||
private int autoAddSize;
|
||||
private ReftableConfig reftableConfig;
|
||||
|
||||
private RevWalk rw;
|
||||
private RevFlag added;
|
||||
|
@ -114,9 +121,19 @@ public DfsPackCompactor(DfsRepository repository) {
|
|||
repo = repository;
|
||||
autoAddSize = 5 * 1024 * 1024; // 5 MiB
|
||||
srcPacks = new ArrayList<>();
|
||||
srcReftables = new ArrayList<>();
|
||||
exclude = new ArrayList<>(4);
|
||||
newPacks = new ArrayList<>(1);
|
||||
newStats = new ArrayList<>(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param cfg
|
||||
* configuration to write a reftable. Reftable compacting is
|
||||
* disabled (default) when {@code cfg} is {@code null}.
|
||||
* @return {@code this}
|
||||
*/
|
||||
public DfsPackCompactor setReftableConfig(ReftableConfig cfg) {
|
||||
reftableConfig = cfg;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -137,7 +154,19 @@ public DfsPackCompactor add(DfsPackFile pack) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Automatically select packs to be included, and add them.
|
||||
* Add a reftable to be compacted.
|
||||
*
|
||||
* @param table
|
||||
* a reftable to combine.
|
||||
* @return {@code this}
|
||||
*/
|
||||
public DfsPackCompactor add(DfsReftable table) {
|
||||
srcReftables.add(table);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatically select pack and reftables to be included, and add them.
|
||||
* <p>
|
||||
* Packs are selected based on size, smaller packs get included while bigger
|
||||
* ones are omitted.
|
||||
|
@ -155,6 +184,16 @@ public DfsPackCompactor autoAdd() throws IOException {
|
|||
else
|
||||
exclude(pack);
|
||||
}
|
||||
|
||||
if (reftableConfig != null) {
|
||||
for (DfsReftable table : objdb.getReftables()) {
|
||||
DfsPackDescription d = table.getPackDescription();
|
||||
if (d.getPackSource() != GC
|
||||
&& d.getFileSize(REFTABLE) < autoAddSize) {
|
||||
add(table);
|
||||
}
|
||||
}
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
|
@ -197,11 +236,30 @@ public DfsPackCompactor exclude(DfsPackFile pack) throws IOException {
|
|||
* the packs cannot be compacted.
|
||||
*/
|
||||
public void compact(ProgressMonitor pm) throws IOException {
|
||||
if (pm == null)
|
||||
if (pm == null) {
|
||||
pm = NullProgressMonitor.INSTANCE;
|
||||
}
|
||||
|
||||
DfsObjDatabase objdb = repo.getObjectDatabase();
|
||||
try (DfsReader ctx = objdb.newReader()) {
|
||||
if (reftableConfig != null && !srcReftables.isEmpty()) {
|
||||
compactReftables(ctx);
|
||||
}
|
||||
compactPacks(ctx, pm);
|
||||
|
||||
List<DfsPackDescription> commit = getNewPacks();
|
||||
Collection<DfsPackDescription> remove = toPrune();
|
||||
if (!commit.isEmpty() || !remove.isEmpty()) {
|
||||
objdb.commitPack(commit, remove);
|
||||
}
|
||||
} finally {
|
||||
rw = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void compactPacks(DfsReader ctx, ProgressMonitor pm)
|
||||
throws IOException, IncorrectObjectTypeException {
|
||||
DfsObjDatabase objdb = repo.getObjectDatabase();
|
||||
PackConfig pc = new PackConfig(repo);
|
||||
pc.setIndexVersion(2);
|
||||
pc.setDeltaCompress(false);
|
||||
|
@ -215,40 +273,31 @@ public void compact(ProgressMonitor pm) throws IOException {
|
|||
|
||||
addObjectsToPack(pw, ctx, pm);
|
||||
if (pw.getObjectCount() == 0) {
|
||||
List<DfsPackDescription> remove = toPrune();
|
||||
if (remove.size() > 0)
|
||||
objdb.commitPack(
|
||||
Collections.<DfsPackDescription>emptyList(),
|
||||
remove);
|
||||
return;
|
||||
}
|
||||
|
||||
boolean rollback = true;
|
||||
DfsPackDescription pack = objdb.newPack(COMPACT,
|
||||
estimatePackSize());
|
||||
initOutDesc(objdb);
|
||||
try {
|
||||
writePack(objdb, pack, pw, pm);
|
||||
writeIndex(objdb, pack, pw);
|
||||
writePack(objdb, outDesc, pw, pm);
|
||||
writeIndex(objdb, outDesc, pw);
|
||||
|
||||
PackStatistics stats = pw.getStatistics();
|
||||
pw.close();
|
||||
pw = null;
|
||||
|
||||
pack.setPackStats(stats);
|
||||
objdb.commitPack(Collections.singletonList(pack), toPrune());
|
||||
newPacks.add(pack);
|
||||
newStats.add(stats);
|
||||
outDesc.setPackStats(stats);
|
||||
newStats = stats;
|
||||
rollback = false;
|
||||
} finally {
|
||||
if (rollback)
|
||||
objdb.rollbackPack(Collections.singletonList(pack));
|
||||
if (rollback) {
|
||||
objdb.rollbackPack(Collections.singletonList(outDesc));
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (pw != null)
|
||||
if (pw != null) {
|
||||
pw.close();
|
||||
}
|
||||
} finally {
|
||||
rw = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -263,27 +312,81 @@ private long estimatePackSize() {
|
|||
return size;
|
||||
}
|
||||
|
||||
private void compactReftables(DfsReader ctx) throws IOException {
|
||||
DfsObjDatabase objdb = repo.getObjectDatabase();
|
||||
Collections.sort(srcReftables, objdb.reftableComparator());
|
||||
|
||||
try (ReftableStack stack = ReftableStack.open(ctx, srcReftables)) {
|
||||
initOutDesc(objdb);
|
||||
ReftableCompactor compact = new ReftableCompactor();
|
||||
compact.addAll(stack.readers());
|
||||
compact.setIncludeDeletes(true);
|
||||
writeReftable(objdb, outDesc, compact);
|
||||
}
|
||||
}
|
||||
|
||||
private void initOutDesc(DfsObjDatabase objdb) throws IOException {
|
||||
if (outDesc == null) {
|
||||
outDesc = objdb.newPack(COMPACT, estimatePackSize());
|
||||
}
|
||||
}
|
||||
|
||||
/** @return all of the source packs that fed into this compaction. */
|
||||
public List<DfsPackDescription> getSourcePacks() {
|
||||
return toPrune();
|
||||
public Collection<DfsPackDescription> getSourcePacks() {
|
||||
Set<DfsPackDescription> src = new HashSet<>();
|
||||
for (DfsPackFile pack : srcPacks) {
|
||||
src.add(pack.getPackDescription());
|
||||
}
|
||||
for (DfsReftable table : srcReftables) {
|
||||
src.add(table.getPackDescription());
|
||||
}
|
||||
return src;
|
||||
}
|
||||
|
||||
/** @return new packs created by this compaction. */
|
||||
public List<DfsPackDescription> getNewPacks() {
|
||||
return newPacks;
|
||||
return outDesc != null
|
||||
? Collections.singletonList(outDesc)
|
||||
: Collections.emptyList();
|
||||
}
|
||||
|
||||
/** @return statistics corresponding to the {@link #getNewPacks()}. */
|
||||
public List<PackStatistics> getNewPackStatistics() {
|
||||
return newStats;
|
||||
return newStats != null
|
||||
? Collections.singletonList(newStats)
|
||||
: Collections.emptyList();
|
||||
}
|
||||
|
||||
private List<DfsPackDescription> toPrune() {
|
||||
int cnt = srcPacks.size();
|
||||
List<DfsPackDescription> all = new ArrayList<>(cnt);
|
||||
for (DfsPackFile pack : srcPacks)
|
||||
all.add(pack.getPackDescription());
|
||||
return all;
|
||||
private Collection<DfsPackDescription> toPrune() {
|
||||
Set<DfsPackDescription> packs = new HashSet<>();
|
||||
for (DfsPackFile pack : srcPacks) {
|
||||
packs.add(pack.getPackDescription());
|
||||
}
|
||||
|
||||
Set<DfsPackDescription> reftables = new HashSet<>();
|
||||
for (DfsReftable table : srcReftables) {
|
||||
reftables.add(table.getPackDescription());
|
||||
}
|
||||
|
||||
for (Iterator<DfsPackDescription> i = packs.iterator(); i.hasNext();) {
|
||||
DfsPackDescription d = i.next();
|
||||
if (d.hasFileExt(REFTABLE) && !reftables.contains(d)) {
|
||||
i.remove();
|
||||
}
|
||||
}
|
||||
|
||||
for (Iterator<DfsPackDescription> i = reftables.iterator();
|
||||
i.hasNext();) {
|
||||
DfsPackDescription d = i.next();
|
||||
if (d.hasFileExt(PACK) && !packs.contains(d)) {
|
||||
i.remove();
|
||||
}
|
||||
}
|
||||
|
||||
Set<DfsPackDescription> toPrune = new HashSet<>();
|
||||
toPrune.addAll(packs);
|
||||
toPrune.addAll(reftables);
|
||||
return toPrune;
|
||||
}
|
||||
|
||||
private void addObjectsToPack(PackWriter pw, DfsReader ctx,
|
||||
|
@ -390,6 +493,27 @@ private static void writeIndex(DfsObjDatabase objdb,
|
|||
}
|
||||
}
|
||||
|
||||
private void writeReftable(DfsObjDatabase objdb, DfsPackDescription pack,
|
||||
ReftableCompactor compact) throws IOException {
|
||||
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
|
||||
compact.setConfig(configureReftable(reftableConfig, out));
|
||||
compact.compact(out);
|
||||
pack.addFileExt(REFTABLE);
|
||||
pack.setReftableStats(compact.getStats());
|
||||
}
|
||||
}
|
||||
|
||||
static ReftableConfig configureReftable(ReftableConfig cfg,
|
||||
DfsOutputStream out) {
|
||||
int bs = out.blockSize();
|
||||
if (bs > 0) {
|
||||
cfg = new ReftableConfig(cfg);
|
||||
cfg.setRefBlockSize(bs);
|
||||
cfg.setAlignBlocks(true);
|
||||
}
|
||||
return cfg;
|
||||
}
|
||||
|
||||
private static class ObjectIdWithOffset extends ObjectId {
|
||||
final long offset;
|
||||
|
||||
|
|
|
@ -159,9 +159,16 @@ public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) {
|
|||
* tables to compact. Tables should be ordered oldest first/most
|
||||
* recent last so that the more recent tables can shadow the
|
||||
* older results. Caller is responsible for closing the readers.
|
||||
* @throws IOException
|
||||
* update indexes of a reader cannot be accessed.
|
||||
*/
|
||||
public void addAll(List<? extends Reftable> readers) {
|
||||
public void addAll(List<? extends Reftable> readers) throws IOException {
|
||||
tables.addAll(readers);
|
||||
for (Reftable r : readers) {
|
||||
if (r instanceof ReftableReader) {
|
||||
adjustUpdateIndexes((ReftableReader) r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -178,7 +185,7 @@ public void addAll(List<? extends Reftable> readers) {
|
|||
* @return {@code true} if the compactor accepted this table; {@code false}
|
||||
* if the compactor has reached its limit.
|
||||
* @throws IOException
|
||||
* if size of {@code reader} cannot be read.
|
||||
* if size of {@code reader}, or its update indexes cannot be read.
|
||||
*/
|
||||
public boolean tryAddFirst(ReftableReader reader) throws IOException {
|
||||
long sz = reader.size();
|
||||
|
@ -186,10 +193,20 @@ public boolean tryAddFirst(ReftableReader reader) throws IOException {
|
|||
return false;
|
||||
}
|
||||
bytesToCompact += sz;
|
||||
adjustUpdateIndexes(reader);
|
||||
tables.addFirst(reader);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void adjustUpdateIndexes(ReftableReader reader) throws IOException {
|
||||
if (minUpdateIndex == 0) {
|
||||
minUpdateIndex = reader.minUpdateIndex();
|
||||
} else {
|
||||
minUpdateIndex = Math.min(minUpdateIndex, reader.minUpdateIndex());
|
||||
}
|
||||
maxUpdateIndex = Math.max(maxUpdateIndex, reader.maxUpdateIndex());
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a compaction to {@code out}.
|
||||
*
|
||||
|
|
Loading…
Reference in New Issue