diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index 4f6d249c5..5a91e173b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -44,12 +44,15 @@ package org.eclipse.jgit.internal.storage.file; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Iterator; +import java.util.List; import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.eclipse.jgit.revwalk.RevCommit; @@ -132,7 +135,8 @@ public void testPackAllObjectsInOnePack(boolean aggressive) } @Theory - public void testPackCommitsAndLooseOne(boolean aggressive) throws Exception { + public void testPackCommitsAndLooseOne(boolean aggressive) + throws Exception { BranchBuilder bb = tr.branch("refs/heads/master"); RevCommit first = bb.commit().add("A", "A").add("B", "B").create(); bb.commit().add("A", "A2").add("B", "B2").create(); @@ -226,71 +230,139 @@ public void testDonePruneTooYoungPacks() throws Exception { } @Test - public void testCommitRangeForBitmaps() throws Exception { - BranchBuilder bb1 = tr.branch("refs/heads/master"); - bb1.commit().message("A1").add("A1", "A1").create(); - bb1.commit().message("B1").add("B1", "B1").create(); - bb1.commit().message("C1").add("C1", "C1").create(); - BranchBuilder bb2 = tr.branch("refs/heads/working"); - bb2.commit().message("A2").add("A2", "A2").create(); - bb2.commit().message("B2").add("B2", "B2").create(); - bb2.commit().message("C2").add("C2", "C2").create(); + public void testBitmapSpansNoMerges() throws Exception { + /* + * Commit counts -> expected bitmap counts for history without merges. + * The top 100 contiguous commits should always have bitmaps, and the + * "recent" bitmaps beyond that are spaced out every 100-200 commits. + * (Starting at 100, the next 100 commits are searched for a merge + * commit. Since one is not found, the spacing between commits is 200. + */ + int[][] bitmapCounts = { // + { 1, 1 }, { 50, 50 }, { 99, 99 }, { 100, 100 }, { 101, 100 }, + { 200, 100 }, { 201, 100 }, { 299, 100 }, { 300, 101 }, + { 301, 101 }, { 401, 101 }, { 499, 101 }, { 500, 102 }, }; + int currentCommits = 0; + BranchBuilder bb = tr.branch("refs/heads/main"); - // Consider all commits. Since history isn't deep all commits are - // selected. - configureGcRange(gc, -1); - gc.gc(); - assertEquals(6, gc.getStatistics().numberOfBitmaps); + for (int[] counts : bitmapCounts) { + int nextCommitCount = counts[0]; + int expectedBitmapCount = counts[1]; + assertTrue(nextCommitCount > currentCommits); // programming error + for (int i = currentCommits; i < nextCommitCount; i++) { + String str = "A" + i; + bb.commit().message(str).add(str, str).create(); + } + currentCommits = nextCommitCount; - // Range==0 means don't examine commit history, create bitmaps only for - // branch tips, C1 & C2. - configureGcRange(gc, 0); - gc.gc(); - assertEquals(2, gc.getStatistics().numberOfBitmaps); - - // Consider only the most recent commit (C2, which is also a branch - // tip). - configureGcRange(gc, 1); - gc.gc(); - assertEquals(2, gc.getStatistics().numberOfBitmaps); - - // Consider only the two most recent commits, C2 & B2. C1 gets included - // too since it is a branch tip. - configureGcRange(gc, 2); - gc.gc(); - assertEquals(3, gc.getStatistics().numberOfBitmaps); - - // Consider C2 & B2 & A2. C1 gets included too since it is a branch tip. - configureGcRange(gc, 3); - gc.gc(); - assertEquals(4, gc.getStatistics().numberOfBitmaps); - - // Consider C2 & B2 & A2 & C1. - configureGcRange(gc, 4); - gc.gc(); - assertEquals(4, gc.getStatistics().numberOfBitmaps); - - // Consider C2 & B2 & A2 & C1 & B1. - configureGcRange(gc, 5); - gc.gc(); - assertEquals(5, gc.getStatistics().numberOfBitmaps); - - // Consider all six commits. - configureGcRange(gc, 6); - gc.gc(); - assertEquals(6, gc.getStatistics().numberOfBitmaps); - - // Input is out of range but should be capped to the total number of - // commits. - configureGcRange(gc, 1000); - gc.gc(); - assertEquals(6, gc.getStatistics().numberOfBitmaps); + gc.setExpireAgeMillis(0); // immediately delete old packs + gc.gc(); + assertEquals(currentCommits * 3, // commit/tree/object + gc.getStatistics().numberOfPackedObjects); + assertEquals(currentCommits + " commits: ", expectedBitmapCount, + gc.getStatistics().numberOfBitmaps); + } } - private void configureGcRange(GC myGc, int range) { - PackConfig pconfig = new PackConfig(repo); - pconfig.setBitmapCommitRange(range); - myGc.setPackConfig(pconfig); + @Test + public void testBitmapSpansWithMerges() throws Exception { + /* + * Commits that are merged. Since 55 is in the oldest history it is + * never considered. Searching goes from oldest to newest so 115 is the + * first merge commit found. After that the range 116-216 is ignored so + * 175 is never considered. + */ + List merges = Arrays.asList(Integer.valueOf(55), + Integer.valueOf(115), Integer.valueOf(175), + Integer.valueOf(235)); + /* + * Commit counts -> expected bitmap counts for history with merges. The + * top 100 contiguous commits should always have bitmaps, and the + * "recent" bitmaps beyond that are spaced out every 100-200 commits. + * Merges in the < 100 range have no effect and merges in the > 100 + * range will only be considered for commit counts > 200. + */ + int[][] bitmapCounts = { // + { 1, 1 }, { 55, 55 }, { 56, 57 }, // +1 bitmap from branch A55 + { 99, 100 }, // still +1 branch @55 + { 100, 100 }, // 101 commits, only 100 newest + { 116, 100 }, // @55 still in 100 newest bitmaps + { 176, 101 }, // @55 branch tip is not in 100 newest + { 213, 101 }, // 216 commits, @115&@175 in 100 newest + { 214, 102 }, // @55 branch tip, merge @115, @177 in newest + { 236, 102 }, // all 4 merge points in history + { 273, 102 }, // 277 commits, @175&@235 in newest + { 274, 103 }, // @55, @115, merge @175, @235 in newest + { 334, 103 }, // @55,@115,@175, @235 in newest + { 335, 104 }, // @55,@115,@175, merge @235 + { 435, 104 }, // @55,@115,@175,@235 tips + { 436, 104 }, // force @236 + }; + + int currentCommits = 0; + BranchBuilder bb = tr.branch("refs/heads/main"); + + for (int[] counts : bitmapCounts) { + int nextCommitCount = counts[0]; + int expectedBitmapCount = counts[1]; + assertTrue(nextCommitCount > currentCommits); // programming error + for (int i = currentCommits; i < nextCommitCount; i++) { + String str = "A" + i; + if (!merges.contains(Integer.valueOf(i))) { + bb.commit().message(str).add(str, str).create(); + } else { + BranchBuilder bbN = tr.branch("refs/heads/A" + i); + bb.commit().message(str).add(str, str) + .parent(bbN.commit().create()).create(); + } + } + currentCommits = nextCommitCount; + + gc.setExpireAgeMillis(0); // immediately delete old packs + gc.gc(); + assertEquals(currentCommits + " commits: ", expectedBitmapCount, + gc.getStatistics().numberOfBitmaps); + } + } + + @Test + public void testBitmapsForExcessiveBranches() throws Exception { + int oneDayInSeconds = 60 * 60 * 24; + + // All of branch A is committed on day1 + BranchBuilder bbA = tr.branch("refs/heads/A"); + for (int i = 0; i < 1001; i++) { + String msg = "A" + i; + bbA.commit().message(msg).add(msg, msg).create(); + } + // All of in branch B is committed on day91 + tr.tick(oneDayInSeconds * 90); + BranchBuilder bbB = tr.branch("refs/heads/B"); + for (int i = 0; i < 1001; i++) { + String msg = "B" + i; + bbB.commit().message(msg).add(msg, msg).create(); + } + // Create 100 other branches with a single commit + for (int i = 0; i < 100; i++) { + BranchBuilder bb = tr.branch("refs/heads/N" + i); + String msg = "singlecommit" + i; + bb.commit().message(msg).add(msg, msg).create(); + } + // now is day92 + tr.tick(oneDayInSeconds); + + // Since there are no merges, commits in recent history are selected + // every 200 commits. + final int commitsForSparseBranch = 1 + (1001 / 200); + final int commitsForFullBranch = 100 + (901 / 200); + final int commitsForShallowBranches = 100; + + // Excessive branch history pruning, one old branch. + gc.gc(); + assertEquals( + commitsForSparseBranch + commitsForFullBranch + + commitsForShallowBranches, + gc.getStatistics().numberOfBitmaps); } private void configureGc(GC myGc, boolean aggressive) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java index a764f0fdd..5abf62548 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java @@ -52,6 +52,7 @@ import org.eclipse.jgit.junit.TestRepository.CommitBuilder; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; import org.junit.After; import org.junit.Before; @@ -65,7 +66,8 @@ public abstract class GcTestCase extends LocalDiskRepositoryTestCase { public void setUp() throws Exception { super.setUp(); repo = createWorkRepository(); - tr = new TestRepository((repo)); + tr = new TestRepository(repo, new RevWalk(repo), + mockSystemReader); gc = new GC(repo); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index cb6907453..8a8c173c2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -2017,11 +2017,8 @@ public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException { PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer( reader, writeBitmaps, pm, stats.interestingObjects); - int commitRange = config.getBitmapCommitRange(); - if (commitRange < 0) - commitRange = numCommits; // select from all commits Collection selectedCommits = - bitmapPreparer.doCommitSelection(commitRange); + bitmapPreparer.selectCommits(numCommits); beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index efde4792e..9bb8789f6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -55,8 +55,6 @@ import java.util.List; import java.util.Set; -import com.googlecode.javaewah.EWAHCompressedBitmap; - import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; @@ -75,14 +73,18 @@ import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.util.BlockList; +import org.eclipse.jgit.util.SystemReader; -/** Helper class for the PackWriter to select commits for pack index bitmaps. */ +/** + * Helper class for the {@link PackWriter} to select commits for which to build + * pack index bitmaps. + */ class PackWriterBitmapPreparer { - private static final Comparator BUILDER_BY_CARDINALITY_DSC = - new Comparator() { - public int compare(BitmapBuilder a, BitmapBuilder b) { - return Integer.signum(b.cardinality() - a.cardinality()); + private static final Comparator ORDER_BY_DESCENDING_CARDINALITY = new Comparator() { + public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) { + return Integer.signum(b.getBuilder().cardinality() + - a.getBuilder().cardinality()); } }; @@ -93,8 +95,13 @@ public int compare(BitmapBuilder a, BitmapBuilder b) { private final BitmapIndexImpl commitBitmapIndex; private final PackBitmapIndexRemapper bitmapRemapper; private final BitmapIndexImpl bitmapIndex; - private final int minCommits = 100; - private final int maxCommits = 5000; + + private final int contiguousCommitCount; + private final int recentCommitCount; + private final int recentCommitSpan; + private final int distantCommitSpan; + private final int excessiveBranchCount; + private final long inactiveBranchTimestamp; PackWriterBitmapPreparer(ObjectReader reader, PackBitmapIndexBuilder writeBitmaps, ProgressMonitor pm, @@ -107,69 +114,138 @@ public int compare(BitmapBuilder a, BitmapBuilder b) { this.bitmapRemapper = PackBitmapIndexRemapper.newPackBitmapIndex( reader.getBitmapIndex(), writeBitmaps); this.bitmapIndex = new BitmapIndexImpl(bitmapRemapper); + this.contiguousCommitCount = 100; + this.recentCommitCount = 20000; + this.recentCommitSpan = 100; + this.distantCommitSpan = 5000; + this.excessiveBranchCount = 100; + long now = SystemReader.getInstance().getCurrentTime(); + long ageInSeconds = 90 * 24 * 60 * 60; + this.inactiveBranchTimestamp = (now / 1000) - ageInSeconds; } - Collection doCommitSelection(int commitRange) - throws MissingObjectException, IncorrectObjectTypeException, - IOException { + /** + * Returns the commit objects for which bitmap indices should be built. + * + * @param expectedCommitCount + * count of commits in the pack + * @return commit objects for which bitmap indices should be built + * @throws IncorrectObjectTypeException + * if any of the processed objects is not a commit + * @throws IOException + * on errors reading pack or index files + * @throws MissingObjectException + * if an expected object is missing + */ + Collection selectCommits(int expectedCommitCount) + throws IncorrectObjectTypeException, IOException, + MissingObjectException { + /* + * Thinking of bitmap indices as a cache, if we find bitmaps at or at a + * close ancestor to 'old' and 'new' when calculating old..new, then all + * objects can be calculated with minimal graph walking. A distribution + * that favors creating bitmaps for the most recent commits maximizes + * the cache hits for clients that are close to HEAD, which is the + * majority of calculations performed. + */ pm.beginTask(JGitText.get().selectingCommits, ProgressMonitor.UNKNOWN); RevWalk rw = new RevWalk(reader); rw.setRetainBody(false); - WalkResult result = findPaths(rw, commitRange); + CommitSelectionHelper selectionHelper = setupTipCommitBitmaps(rw, + expectedCommitCount); pm.endTask(); - int totCommits = result.commitsByOldest.length - result.commitStartPos; + int totCommits = selectionHelper.getCommitCount(); BlockList selections = new BlockList( - totCommits / minCommits + 1); - for (BitmapCommit reuse : result.reuse) + totCommits / recentCommitSpan + 1); + for (BitmapCommit reuse : selectionHelper.reusedCommits) { selections.add(reuse); + } if (totCommits == 0) { - for (AnyObjectId id : result.peeledWant) + for (AnyObjectId id : selectionHelper.peeledWants) { selections.add(new BitmapCommit(id, false, 0)); + } return selections; } pm.beginTask(JGitText.get().selectingCommits, totCommits); + int totalWants = selectionHelper.peeledWants.size(); - for (BitmapBuilder bitmapableCommits : result.paths) { - int cardinality = bitmapableCommits.cardinality(); + for (BitmapBuilderEntry entry : selectionHelper.tipCommitBitmaps) { + BitmapBuilder bitmap = entry.getBuilder(); + int cardinality = bitmap.cardinality(); List> running = new ArrayList< List>(); + // Mark the current branch as inactive if its tip commit isn't + // recent and there are an excessive number of branches, to + // prevent memory bloat of computing too many bitmaps for stale + // branches. + boolean isActiveBranch = true; + if (totalWants > excessiveBranchCount + && !isRecentCommit(entry.getCommit())) { + isActiveBranch = false; + } + // Insert bitmaps at the offsets suggested by the // nextSelectionDistance() heuristic. int index = -1; - int nextIn = nextSelectionDistance(0, cardinality); - int nextFlg = nextIn == maxCommits ? PackBitmapIndex.FLAG_REUSE : 0; - boolean mustPick = nextIn == 0; - for (RevCommit c : result) { - if (!bitmapableCommits.contains(c)) + int nextIn = nextSpan(cardinality); + int nextFlg = nextIn == distantCommitSpan + ? PackBitmapIndex.FLAG_REUSE : 0; + + // For the current branch, iterate through all commits from oldest + // to newest. + for (RevCommit c : selectionHelper) { + // Optimization: if we have found all the commits for this + // branch, stop searching + int distanceFromTip = cardinality - index - 1; + if (distanceFromTip == 0) { + break; + } + + // Ignore commits that are not in this branch + if (!bitmap.contains(c)) { continue; + } index++; nextIn--; pm.update(1); - // Always pick the items in want and prefer merge commits. - if (result.peeledWant.remove(c)) { - if (nextIn > 0) + // Always pick the items in wants, prefer merge commits. + if (selectionHelper.peeledWants.remove(c)) { + if (nextIn > 0) { nextFlg = 0; - } else if (!mustPick && ((nextIn > 0) - || (c.getParentCount() <= 1 && nextIn > -minCommits))) { - continue; + } + } else { + boolean stillInSpan = nextIn >= 0; + boolean isMergeCommit = c.getParentCount() > 1; + // Force selection if: + // a) we have exhausted the window looking for merges + // b) we are in the top commits of an active branch + // c) we are at a branch tip + boolean mustPick = (nextIn <= -recentCommitSpan) + || (isActiveBranch + && (distanceFromTip <= contiguousCommitCount)) + || (distanceFromTip == 1); // most recent commit + if (!mustPick && (stillInSpan || !isMergeCommit)) { + continue; + } } + // This commit is selected, calculate the next one. int flags = nextFlg; - nextIn = nextSelectionDistance(index, cardinality); - nextFlg = nextIn == maxCommits ? PackBitmapIndex.FLAG_REUSE : 0; - mustPick = nextIn == 0; + nextIn = nextSpan(distanceFromTip); + nextFlg = nextIn == distantCommitSpan + ? PackBitmapIndex.FLAG_REUSE : 0; BitmapBuilder fullBitmap = commitBitmapIndex.newBitmapBuilder(); rw.reset(); rw.markStart(c); - for (AnyObjectId objectId : result.reuse) + for (AnyObjectId objectId : selectionHelper.reusedCommits) rw.markUninteresting(rw.parseCommit(objectId)); rw.setRevFilter( PackWriterBitmapWalker.newRevFilter(null, fullBitmap)); @@ -182,8 +258,9 @@ Collection doCommitSelection(int commitRange) List>(); for (List list : running) { BitmapCommit last = list.get(list.size() - 1); - if (fullBitmap.contains(last)) + if (fullBitmap.contains(last)) { matches.add(list); + } } List match; @@ -194,121 +271,176 @@ Collection doCommitSelection(int commitRange) match = matches.get(0); // Append to longest for (List list : matches) { - if (list.size() > match.size()) + if (list.size() > match.size()) { match = list; + } } } match.add(new BitmapCommit(c, !match.isEmpty(), flags)); writeBitmaps.addBitmap(c, fullBitmap, 0); } - for (List list : running) + for (List list : running) { selections.addAll(list); + } } writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps. // Add the remaining peeledWant - for (AnyObjectId remainingWant : result.peeledWant) + for (AnyObjectId remainingWant : selectionHelper.peeledWants) { selections.add(new BitmapCommit(remainingWant, false, 0)); + } pm.endTask(); return selections; } - private WalkResult findPaths(RevWalk rw, int commitRange) - throws MissingObjectException, IOException { - BitmapBuilder reuseBitmap = commitBitmapIndex.newBitmapBuilder(); - List reuse = new ArrayList(); + private boolean isRecentCommit(RevCommit revCommit) { + return revCommit.getCommitTime() > inactiveBranchTimestamp; + } + + /** + * For each of the {@code want}s, which represent the tip commit of each + * branch, set up an initial {@link BitmapBuilder}. Reuse previously built + * bitmaps if possible. + * + * @param rw + * a {@link RevWalk} to find reachable objects in this repository + * @param expectedCommitCount + * expected count of commits. The actual count may be less due to + * unreachable garbage. + * @return a {@link CommitSelectionHelper} containing bitmaps for the tip + * commits + * @throws IncorrectObjectTypeException + * if any of the processed objects is not a commit + * @throws IOException + * on errors reading pack or index files + * @throws MissingObjectException + * if an expected object is missing + */ + private CommitSelectionHelper setupTipCommitBitmaps(RevWalk rw, + int expectedCommitCount) throws IncorrectObjectTypeException, + IOException, MissingObjectException { + BitmapBuilder reuse = commitBitmapIndex.newBitmapBuilder(); + List reuseCommits = new ArrayList(); for (PackBitmapIndexRemapper.Entry entry : bitmapRemapper) { - if ((entry.getFlags() & FLAG_REUSE) != FLAG_REUSE) + if ((entry.getFlags() & FLAG_REUSE) != FLAG_REUSE) { continue; + } RevObject ro = rw.peel(rw.parseAny(entry)); if (ro instanceof RevCommit) { RevCommit rc = (RevCommit) ro; - reuse.add(new BitmapCommit(rc, false, entry.getFlags())); + reuseCommits.add(new BitmapCommit(rc, false, entry.getFlags())); rw.markUninteresting(rc); - - EWAHCompressedBitmap bitmap = bitmapRemapper.ofObjectType( - bitmapRemapper.getBitmap(rc), Constants.OBJ_COMMIT); - writeBitmaps.addBitmap(rc, bitmap, 0); - reuseBitmap.add(rc, Constants.OBJ_COMMIT); + // PackBitmapIndexRemapper.ofObjectType() ties the underlying + // bitmap in the old pack into the new bitmap builder. + bitmapRemapper.ofObjectType(bitmapRemapper.getBitmap(rc), + Constants.OBJ_COMMIT).trim(); + reuse.add(rc, Constants.OBJ_COMMIT); } } - writeBitmaps.clearBitmaps(); // Remove temporary bitmaps // Do a RevWalk by commit time descending. Keep track of all the paths // from the wants. - List paths = new ArrayList(want.size()); + List tipCommitBitmaps = new ArrayList( + want.size()); Set peeledWant = new HashSet(want.size()); for (AnyObjectId objectId : want) { RevObject ro = rw.peel(rw.parseAny(objectId)); - if (ro instanceof RevCommit && !reuseBitmap.contains(ro)) { + if (ro instanceof RevCommit && !reuse.contains(ro)) { RevCommit rc = (RevCommit) ro; peeledWant.add(rc); rw.markStart(rc); BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder(); - bitmap.or(reuseBitmap); + bitmap.or(reuse); bitmap.add(rc, Constants.OBJ_COMMIT); - paths.add(bitmap); + tipCommitBitmaps.add(new BitmapBuilderEntry(rc, bitmap)); } } - // Update the paths from the wants and create a list of commits in - // reverse iteration order for the desired commit range. - RevCommit[] commits = new RevCommit[commitRange]; + // Create a list of commits in reverse order (older to newer). + RevCommit[] commits = new RevCommit[expectedCommitCount]; int pos = commits.length; RevCommit rc; while ((rc = rw.next()) != null && pos > 0) { commits[--pos] = rc; - for (BitmapBuilder path : paths) { - if (path.contains(rc)) { - for (RevCommit c : rc.getParents()) - path.add(c, Constants.OBJ_COMMIT); + for (BitmapBuilderEntry entry : tipCommitBitmaps) { + BitmapBuilder bitmap = entry.getBuilder(); + if (bitmap.contains(rc)) { + for (RevCommit c : rc.getParents()) { + bitmap.add(c, Constants.OBJ_COMMIT); + } } } - pm.update(1); } - // Remove the reused bitmaps from the paths - if (!reuse.isEmpty()) - for (BitmapBuilder bitmap : paths) - bitmap.andNot(reuseBitmap); + // Remove the reused bitmaps from the tip commit bitmaps + if (!reuseCommits.isEmpty()) { + for (BitmapBuilderEntry entry : tipCommitBitmaps) { + entry.getBuilder().andNot(reuse); + } + } - // Sort the paths - List distinctPaths = new ArrayList(paths.size()); - while (!paths.isEmpty()) { - Collections.sort(paths, BUILDER_BY_CARDINALITY_DSC); - BitmapBuilder largest = paths.remove(0); - distinctPaths.add(largest); + // Sort the tip commit bitmaps. Find the one containing the most + // commits, remove those commits from the remaining bitmaps, resort and + // repeat. + List orderedTipCommitBitmaps = new ArrayList<>( + tipCommitBitmaps.size()); + while (!tipCommitBitmaps.isEmpty()) { + Collections.sort(tipCommitBitmaps, ORDER_BY_DESCENDING_CARDINALITY); + BitmapBuilderEntry largest = tipCommitBitmaps.remove(0); + orderedTipCommitBitmaps.add(largest); // Update the remaining paths, by removing the objects from // the path that was just added. - for (int i = paths.size() - 1; i >= 0; i--) - paths.get(i).andNot(largest); + for (int i = tipCommitBitmaps.size() - 1; i >= 0; i--) { + tipCommitBitmaps.get(i).getBuilder() + .andNot(largest.getBuilder()); + } } - return new WalkResult(peeledWant, commits, pos, distinctPaths, reuse); + return new CommitSelectionHelper(peeledWant, commits, pos, + orderedTipCommitBitmaps, reuseCommits); } - private int nextSelectionDistance(int idx, int cardinality) { - if (idx > cardinality) + /*- + * Returns the desired distance to the next bitmap based on the distance + * from the tip commit. Only differentiates recent from distant spans, + * selectCommits() handles the contiguous commits at the tip for active + * or inactive branches. + * + * A graph of this function looks like this, where + * the X axis is the distance from the tip commit and the Y axis is the + * bitmap selection distance. + * + * 5000 ____... + * / + * / + * / + * / + * 100 _____/ + * 0 20100 25000 + * + * Linear scaling between 20100 and 25000 prevents spans >100 for distances + * <20000 (otherwise, a span of 5000 would be returned for a distance of + * 21000, and the range 16000-20000 would have no selections). + */ + int nextSpan(int distanceFromTip) { + if (distanceFromTip < 0) { throw new IllegalArgumentException(); - int idxFromStart = cardinality - idx; - int mustRegionEnd = 100; - if (idxFromStart <= mustRegionEnd) - return 0; + } // Commits more toward the start will have more bitmaps. - int minRegionEnd = 20000; - if (idxFromStart <= minRegionEnd) - return Math.min(idxFromStart - mustRegionEnd, minCommits); + if (distanceFromTip <= recentCommitCount) { + return recentCommitSpan; + } - // Commits more toward the end will have fewer. - int next = Math.min(idxFromStart - minRegionEnd, maxCommits); - return Math.max(next, minCommits); + int next = Math.min(distanceFromTip - recentCommitCount, + distantCommitSpan); + return Math.max(next, recentCommitSpan); } PackWriterBitmapWalker newBitmapWalker() { @@ -316,12 +448,14 @@ PackWriterBitmapWalker newBitmapWalker() { new ObjectWalk(reader), bitmapIndex, null); } + /** + * A commit object for which a bitmap index should be built. + */ static final class BitmapCommit extends ObjectId { private final boolean reuseWalker; private final int flags; - private BitmapCommit( - AnyObjectId objectId, boolean reuseWalker, int flags) { + BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags) { super(objectId); this.reuseWalker = reuseWalker; this.flags = flags; @@ -336,21 +470,55 @@ int getFlags() { } } - private static final class WalkResult implements Iterable { - private final Set peeledWant; + /** + * A POJO representing a Pair. + */ + private static final class BitmapBuilderEntry { + private final RevCommit commit; + + private final BitmapBuilder builder; + + BitmapBuilderEntry(RevCommit commit, BitmapBuilder builder) { + this.commit = commit; + this.builder = builder; + } + + RevCommit getCommit() { + return commit; + } + + BitmapBuilder getBuilder() { + return builder; + } + } + + /** + * Container for state used in the first phase of selecting commits, which + * walks all of the reachable commits via the branch tips ( + * {@code peeledWants}), stores them in {@code commitsByOldest}, and sets up + * bitmaps for each branch tip ({@code tipCommitBitmaps}). + * {@code commitsByOldest} is initialized with an expected size of all + * commits, but may be smaller if some commits are unreachable, in which + * case {@code commitStartPos} will contain a positive offset to the root + * commit. + */ + private static final class CommitSelectionHelper implements Iterable { + final Set peeledWants; + + final List tipCommitBitmaps; + final Iterable reusedCommits; private final RevCommit[] commitsByOldest; private final int commitStartPos; - private final List paths; - private final Iterable reuse; - private WalkResult(Set peeledWant, + CommitSelectionHelper(Set peeledWant, RevCommit[] commitsByOldest, int commitStartPos, - List paths, Iterable reuse) { - this.peeledWant = peeledWant; + List bitmapEntries, + Iterable reuse) { + this.peeledWants = peeledWant; this.commitsByOldest = commitsByOldest; this.commitStartPos = commitStartPos; - this.paths = paths; - this.reuse = reuse; + this.tipCommitBitmaps = bitmapEntries; + this.reusedCommits = reuse; } public Iterator iterator() { @@ -370,5 +538,9 @@ public void remove() { } }; } + + int getCommitCount() { + return commitsByOldest.length - commitStartPos; + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 40309962c..a8835b76c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -138,14 +138,6 @@ public class PackConfig { */ public static final boolean DEFAULT_BUILD_BITMAPS = true; - /** - * Default range of commits for which to create bitmaps: {@value} - * - * @see #setBitmapCommitRange(int) - * @since 4.2 - */ - public static final int DEFAULT_BITMAP_COMMIT_RANGE = -1; - private int compressionLevel = Deflater.DEFAULT_COMPRESSION; @@ -177,8 +169,6 @@ public class PackConfig { private boolean buildBitmaps = DEFAULT_BUILD_BITMAPS; - private int bitmapCommitRange = DEFAULT_BITMAP_COMMIT_RANGE; - private boolean cutDeltaChains; /** Create a default configuration. */ @@ -232,7 +222,6 @@ public PackConfig(PackConfig cfg) { this.executor = cfg.executor; this.indexVersion = cfg.indexVersion; this.buildBitmaps = cfg.buildBitmaps; - this.bitmapCommitRange = cfg.bitmapCommitRange; this.cutDeltaChains = cfg.cutDeltaChains; } @@ -694,39 +683,6 @@ public void setBuildBitmaps(boolean buildBitmaps) { this.buildBitmaps = buildBitmaps; } - /** - * Get the range of commits for which to build bitmaps. The range starts - * from the most recent commit. - * - * A value of 0 creates bitmaps for only branch tips. A value of -1 creates - * bitmaps spaced through the entire history of commits. - * - * Default setting: {@value #DEFAULT_BITMAP_COMMIT_RANGE} - * - * @return the range of commits for which to create bitmaps, starting with - * the most recent commit - * @see PackIndexWriter - * @since 4.2 - */ - public int getBitmapCommitRange() { - return bitmapCommitRange; - } - - /** - * Set the range of commits for which to build bitmaps. - * - * Default setting: {@value #DEFAULT_BITMAP_COMMIT_RANGE} - * - * @param range - * the range of commits for which to create bitmaps, starting - * with the most recent commit - * @see PackIndexWriter - * @since 4.2 - */ - public void setBitmapCommitRange(final int range) { - bitmapCommitRange = range; - } - /** * Update properties by setting fields from the configuration. * @@ -762,8 +718,6 @@ public void fromConfig(final Config rc) { setCutDeltaChains(rc.getBoolean( "pack", "cutdeltachains", getCutDeltaChains())); //$NON-NLS-1$ //$NON-NLS-2$ setBuildBitmaps(rc.getBoolean("pack", "buildbitmaps", isBuildBitmaps())); //$NON-NLS-1$ //$NON-NLS-2$ - setBitmapCommitRange( - rc.getInt("pack", "bitmapcommitrange", getBitmapCommitRange())); //$NON-NLS-1$ //$NON-NLS-2$ } public String toString() { @@ -781,7 +735,6 @@ public String toString() { b.append(", reuseObjects=").append(isReuseObjects()); //$NON-NLS-1$ b.append(", deltaCompress=").append(isDeltaCompress()); //$NON-NLS-1$ b.append(", buildBitmaps=").append(isBuildBitmaps()); //$NON-NLS-1$ - b.append(", bitmapCommitRange=").append(getBitmapCommitRange()); //$NON-NLS-1$ return b.toString(); } }