Expose bitmap selection parameters via PackConfig

Expose the following bitmap selection parameters via PackConfig:
"bitmapContiguousCommitCount", "bitmapRecentCommitCount",
"bitmapRecentCommitSpan", "bitmapDistantCommitSpan",
"bitmapExcessiveBranchCount", and "bitmapInactiveBranchAge".

The value of bitmapContiguousCommitCount, whereby bitmaps are
created for the most recent N commits in a branch, has never
been verified. If experiments show that they are not valuable,
then we can simplify the implementation so that there is only
a concept of recent and distant commit history (defined by
"bitmapRecentCommitCount"), and the only controls we need are
"bitmapRecentCommitSpan" and "bitmapDistantCommitSpan".

Change-Id: I288bf3f97d6fbfdfcd5dde2699eff433a7307fb9
Signed-off-by: Terry Parker <tparker@google.com>
This commit is contained in:
Terry Parker 2015-10-19 15:59:47 -07:00
parent 320a4142ad
commit 80c7884ea6
4 changed files with 419 additions and 16 deletions

View File

@ -0,0 +1,136 @@
package org.eclipse.jgit.internal.storage.pack;
import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BITMAP_DISTANT_COMMIT_SPAN;
import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BITMAP_RECENT_COMMIT_COUNT;
import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BITMAP_RECENT_COMMIT_SPAN;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder;
import org.eclipse.jgit.lib.AbbreviatedObjectId;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.storage.pack.PackConfig;
import org.junit.Test;
/** Tests for the {@link PackWriterBitmapPreparer}. */
public class PackWriterBitmapPreparerTest {
private static class StubObjectReader extends ObjectReader {
@Override
public ObjectReader newReader() {
return null;
}
@Override
public Collection<ObjectId> resolve(AbbreviatedObjectId id)
throws IOException {
return null;
}
@Override
public ObjectLoader open(AnyObjectId objectId, int typeHint)
throws MissingObjectException, IncorrectObjectTypeException,
IOException {
return null;
}
@Override
public Set<ObjectId> getShallowCommits() throws IOException {
return null;
}
@Override
public void close() {
// stub
}
}
@Test
public void testNextSelectionDistanceForActiveBranch() throws Exception {
PackWriterBitmapPreparer preparer = newPeparer(
DEFAULT_BITMAP_RECENT_COMMIT_COUNT, // 20000
DEFAULT_BITMAP_RECENT_COMMIT_SPAN, // 100
DEFAULT_BITMAP_DISTANT_COMMIT_SPAN); // 5000
int[][] distancesAndSpans = { { 0, 100 }, { 100, 100 }, { 10000, 100 },
{ 20000, 100 }, { 20100, 100 }, { 20102, 102 }, { 20200, 200 },
{ 22200, 2200 }, { 24999, 4999 }, { 25000, 5000 },
{ 50000, 5000 }, { 1000000, 5000 }, };
for (int[] pair : distancesAndSpans) {
assertEquals(pair[1], preparer.nextSpan(pair[0]));
}
}
@Test
public void testNextSelectionDistanceWithFewerRecentCommits()
throws Exception {
PackWriterBitmapPreparer preparer = newPeparer(1000,
DEFAULT_BITMAP_RECENT_COMMIT_SPAN, // 100
DEFAULT_BITMAP_DISTANT_COMMIT_SPAN); // 5000
int[][] distancesAndSpans = { { 0, 100 }, { 100, 100 }, { 1000, 100 },
{ 1100, 100 }, { 1111, 111 }, { 2000, 1000 }, { 5999, 4999 },
{ 6000, 5000 }, { 10000, 5000 }, { 50000, 5000 },
{ 1000000, 5000 } };
for (int[] pair : distancesAndSpans) {
assertEquals(pair[1], preparer.nextSpan(pair[0]));
}
}
@Test
public void testNextSelectionDistanceWithSmallerRecentSpan()
throws Exception {
PackWriterBitmapPreparer preparer = newPeparer(
DEFAULT_BITMAP_RECENT_COMMIT_COUNT, // 20000
10, // recent span
DEFAULT_BITMAP_DISTANT_COMMIT_SPAN); // 5000
int[][] distancesAndSpans = { { 0, 10 }, { 100, 10 }, { 10000, 10 },
{ 20000, 10 }, { 20010, 10 }, { 20012, 12 }, { 20050, 50 },
{ 20200, 200 }, { 22200, 2200 }, { 24999, 4999 },
{ 25000, 5000 }, { 50000, 5000 }, { 1000000, 5000 } };
for (int[] pair : distancesAndSpans) {
assertEquals(pair[1], preparer.nextSpan(pair[0]));
}
}
@Test
public void testNextSelectionDistanceWithSmallerDistantSpan()
throws Exception {
PackWriterBitmapPreparer preparer = newPeparer(
DEFAULT_BITMAP_RECENT_COMMIT_COUNT, // 20000
DEFAULT_BITMAP_RECENT_COMMIT_SPAN, // 100
1000);
int[][] distancesAndSpans = { { 0, 100 }, { 100, 100 }, { 10000, 100 },
{ 20000, 100 }, { 20100, 100 }, { 20102, 102 }, { 20200, 200 },
{ 20999, 999 }, { 21000, 1000 }, { 22000, 1000 },
{ 25000, 1000 }, { 50000, 1000 }, { 1000000, 1000 } };
for (int[] pair : distancesAndSpans) {
assertEquals(pair[1], preparer.nextSpan(pair[0]));
}
}
private PackWriterBitmapPreparer newPeparer(int recentCount, int recentSpan,
int distantSpan) throws IOException {
List<ObjectToPack> objects = Collections.emptyList();
Set<ObjectId> wants = Collections.emptySet();
PackConfig config = new PackConfig();
config.setBitmapRecentCommitCount(recentCount);
config.setBitmapRecentCommitSpan(recentSpan);
config.setBitmapDistantCommitSpan(distantSpan);
PackBitmapIndexBuilder indexBuilder = new PackBitmapIndexBuilder(
objects);
return new PackWriterBitmapPreparer(new StubObjectReader(),
indexBuilder, null, wants, config);
}
}

View File

@ -2015,7 +2015,7 @@ public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException {
byName = null;
PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(
reader, writeBitmaps, pm, stats.interestingObjects);
reader, writeBitmaps, pm, stats.interestingObjects, config);
Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits =
bitmapPreparer.selectCommits(numCommits);

View File

@ -72,6 +72,7 @@
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.util.BlockList;
import org.eclipse.jgit.util.SystemReader;
@ -81,6 +82,8 @@
*/
class PackWriterBitmapPreparer {
private static final int DAY_IN_SECONDS = 24 * 60 * 60;
private static final Comparator<BitmapBuilderEntry> ORDER_BY_DESCENDING_CARDINALITY = new Comparator<BitmapBuilderEntry>() {
public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) {
return Integer.signum(b.getBuilder().cardinality()
@ -105,7 +108,8 @@ public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) {
PackWriterBitmapPreparer(ObjectReader reader,
PackBitmapIndexBuilder writeBitmaps, ProgressMonitor pm,
Set<? extends ObjectId> want) throws IOException {
Set<? extends ObjectId> want, PackConfig config)
throws IOException {
this.reader = reader;
this.writeBitmaps = writeBitmaps;
this.pm = pm;
@ -114,13 +118,14 @@ public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) {
this.bitmapRemapper = PackBitmapIndexRemapper.newPackBitmapIndex(
reader.getBitmapIndex(), writeBitmaps);
this.bitmapIndex = new BitmapIndexImpl(bitmapRemapper);
this.contiguousCommitCount = 100;
this.recentCommitCount = 20000;
this.recentCommitSpan = 100;
this.distantCommitSpan = 5000;
this.excessiveBranchCount = 100;
this.contiguousCommitCount = config.getBitmapContiguousCommitCount();
this.recentCommitCount = config.getBitmapRecentCommitCount();
this.recentCommitSpan = config.getBitmapRecentCommitSpan();
this.distantCommitSpan = config.getBitmapDistantCommitSpan();
this.excessiveBranchCount = config.getBitmapExcessiveBranchCount();
long now = SystemReader.getInstance().getCurrentTime();
long ageInSeconds = 90 * 24 * 60 * 60;
long ageInSeconds = config.getBitmapInactiveBranchAgeInDays()
* DAY_IN_SECONDS;
this.inactiveBranchTimestamp = (now / 1000) - ageInSeconds;
}

View File

@ -138,6 +138,65 @@ public class PackConfig {
*/
public static final boolean DEFAULT_BUILD_BITMAPS = true;
/**
* Default count of most recent commits to select for bitmaps. Only applies
* when bitmaps are enabled: {@value}
*
* @see #setBitmapContiguousCommitCount(int)
* @since 4.2
*/
public static final int DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT = 100;
/**
* Count at which the span between selected commits changes from
* "bitmapRecentCommitSpan" to "bitmapDistantCommitSpan". Only applies when
* bitmaps are enabled: {@value}
*
* @see #setBitmapRecentCommitCount(int)
* @since 4.2
*/
public static final int DEFAULT_BITMAP_RECENT_COMMIT_COUNT = 20000;
/**
* Default spacing between commits in recent history when selecting commits
* for bitmaps. Only applies when bitmaps are enabled: {@value}
*
* @see #setBitmapRecentCommitSpan(int)
* @since 4.2
*/
public static final int DEFAULT_BITMAP_RECENT_COMMIT_SPAN = 100;
/**
* Default spacing between commits in distant history when selecting commits
* for bitmaps. Only applies when bitmaps are enabled: {@value}
*
* @see #setBitmapDistantCommitSpan(int)
* @since 4.2
*/
public static final int DEFAULT_BITMAP_DISTANT_COMMIT_SPAN = 5000;
/**
* Default count of branches required to activate inactive branch commit
* selection. If the number of branches is less than this then bitmaps for
* the entire commit history of all branches will be created, otherwise
* branches marked as "inactive" will have coverage for only partial
* history: {@value}
*
* @see #setBitmapExcessiveBranchCount(int)
* @since 4.2
*/
public static final int DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT = 100;
/**
* Default age at which a branch is considered inactive. Age is taken as the
* number of days ago that the most recent commit was made to a branch. Only
* affects bitmap processing if bitmaps are enabled and the
* "excessive branch count" has been exceeded: {@value}
*
* @see #setBitmapInactiveBranchAgeInDays(int)
* @since 4.2
*/
public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90;
private int compressionLevel = Deflater.DEFAULT_COMPRESSION;
@ -169,6 +228,18 @@ public class PackConfig {
private boolean buildBitmaps = DEFAULT_BUILD_BITMAPS;
private int bitmapContiguousCommitCount = DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT;
private int bitmapRecentCommitCount = DEFAULT_BITMAP_RECENT_COMMIT_COUNT;
private int bitmapRecentCommitSpan = DEFAULT_BITMAP_RECENT_COMMIT_SPAN;
private int bitmapDistantCommitSpan = DEFAULT_BITMAP_DISTANT_COMMIT_SPAN;
private int bitmapExcessiveBranchCount = DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT;
private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS;
private boolean cutDeltaChains;
/** Create a default configuration. */
@ -222,6 +293,12 @@ public PackConfig(PackConfig cfg) {
this.executor = cfg.executor;
this.indexVersion = cfg.indexVersion;
this.buildBitmaps = cfg.buildBitmaps;
this.bitmapContiguousCommitCount = cfg.bitmapContiguousCommitCount;
this.bitmapRecentCommitCount = cfg.bitmapRecentCommitCount;
this.bitmapRecentCommitSpan = cfg.bitmapRecentCommitSpan;
this.bitmapDistantCommitSpan = cfg.bitmapDistantCommitSpan;
this.bitmapExcessiveBranchCount = cfg.bitmapExcessiveBranchCount;
this.bitmapInactiveBranchAgeInDays = cfg.bitmapInactiveBranchAgeInDays;
this.cutDeltaChains = cfg.cutDeltaChains;
}
@ -650,7 +727,7 @@ public int getIndexVersion() {
* oldest (most compatible) format available for the objects.
* @see PackIndexWriter
*/
public void setIndexVersion(final int version) {
public void setIndexVersion(int version) {
indexVersion = version;
}
@ -683,6 +760,162 @@ public void setBuildBitmaps(boolean buildBitmaps) {
this.buildBitmaps = buildBitmaps;
}
/**
* Get the count of most recent commits for which to build bitmaps.
*
* Default setting: {@value #DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT}
*
* @return the count of most recent commits for which to build bitmaps
* @since 4.2
*/
public int getBitmapContiguousCommitCount() {
return bitmapContiguousCommitCount;
}
/**
* Set the count of most recent commits for which to build bitmaps.
*
* Default setting: {@value #DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT}
*
* @param count
* the count of most recent commits for which to build bitmaps
* @since 4.2
*/
public void setBitmapContiguousCommitCount(int count) {
bitmapContiguousCommitCount = count;
}
/**
* Get the count at which to switch from "bitmapRecentCommitSpan" to
* "bitmapDistantCommitSpan".
*
* Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_COUNT}
*
* @return the count for switching between recent and distant spans
* @since 4.2
*/
public int getBitmapRecentCommitCount() {
return bitmapRecentCommitCount;
}
/**
* Set the count at which to switch from "bitmapRecentCommitSpan" to
* "bitmapDistantCommitSpan".
*
* Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_COUNT}
*
* @param count
* the count for switching between recent and distant spans
* @since 4.2
*/
public void setBitmapRecentCommitCount(int count) {
bitmapRecentCommitCount = count;
}
/**
* Get the span of commits when building bitmaps for recent history.
*
* Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_SPAN}
*
* @return the span of commits when building bitmaps for recent history
* @since 4.2
*/
public int getBitmapRecentCommitSpan() {
return bitmapRecentCommitSpan;
}
/**
* Set the span of commits when building bitmaps for recent history.
*
* Default setting: {@value #DEFAULT_BITMAP_RECENT_COMMIT_SPAN}
*
* @param span
* the span of commits when building bitmaps for recent history
* @since 4.2
*/
public void setBitmapRecentCommitSpan(int span) {
bitmapRecentCommitSpan = span;
}
/**
* Get the span of commits when building bitmaps for distant history.
*
* Default setting: {@value #DEFAULT_BITMAP_DISTANT_COMMIT_SPAN}
*
* @return the span of commits when building bitmaps for distant history
* @since 4.2
*/
public int getBitmapDistantCommitSpan() {
return bitmapDistantCommitSpan;
}
/**
* Set the span of commits when building bitmaps for distant history.
*
* Default setting: {@value #DEFAULT_BITMAP_DISTANT_COMMIT_SPAN}
*
* @param span
* the span of commits when building bitmaps for distant history
* @since 4.2
*/
public void setBitmapDistantCommitSpan(int span) {
bitmapDistantCommitSpan = span;
}
/**
* Get the count of branches deemed "excessive". If the count of branches in
* a repository exceeds this number and bitmaps are enabled, "inactive"
* branches will have fewer bitmaps than "active" branches.
*
* Default setting: {@value #DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT}
*
* @return the count of branches deemed "excessive"
* @since 4.2
*/
public int getBitmapExcessiveBranchCount() {
return bitmapExcessiveBranchCount;
}
/**
* Set the count of branches deemed "excessive". If the count of branches in
* a repository exceeds this number and bitmaps are enabled, "inactive"
* branches will have fewer bitmaps than "active" branches.
*
* Default setting: {@value #DEFAULT_BITMAP_EXCESSIVE_BRANCH_COUNT}
*
* @param count
* the count of branches deemed "excessive"
* @since 4.2
*/
public void setBitmapExcessiveBranchCount(int count) {
bitmapExcessiveBranchCount = count;
}
/**
* Get the the age in days that marks a branch as "inactive".
*
* Default setting: {@value #DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS}
*
* @return the age in days that marks a branch as "inactive"
* @since 4.2
*/
public int getBitmapInactiveBranchAgeInDays() {
return bitmapInactiveBranchAgeInDays;
}
/**
* Set the the age in days that marks a branch as "inactive".
*
* Default setting: {@value #DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS}
*
* @param ageInDays
* the age in days that marks a branch as "inactive"
* @since 4.2
*/
public void setBitmapInactiveBranchAgeInDays(int ageInDays) {
bitmapInactiveBranchAgeInDays = ageInDays;
}
/**
* Update properties by setting fields from the configuration.
*
@ -712,19 +945,36 @@ public void fromConfig(final Config rc) {
// These variables aren't standardized
//
setReuseDeltas(rc.getBoolean("pack", "reusedeltas", isReuseDeltas())); //$NON-NLS-1$ //$NON-NLS-2$
setReuseObjects(rc.getBoolean("pack", "reuseobjects", isReuseObjects())); //$NON-NLS-1$ //$NON-NLS-2$
setDeltaCompress(rc.getBoolean(
"pack", "deltacompression", isDeltaCompress())); //$NON-NLS-1$ //$NON-NLS-2$
setCutDeltaChains(rc.getBoolean(
"pack", "cutdeltachains", getCutDeltaChains())); //$NON-NLS-1$ //$NON-NLS-2$
setBuildBitmaps(rc.getBoolean("pack", "buildbitmaps", isBuildBitmaps())); //$NON-NLS-1$ //$NON-NLS-2$
setReuseObjects(
rc.getBoolean("pack", "reuseobjects", isReuseObjects())); //$NON-NLS-1$ //$NON-NLS-2$
setDeltaCompress(
rc.getBoolean("pack", "deltacompression", isDeltaCompress())); //$NON-NLS-1$ //$NON-NLS-2$
setCutDeltaChains(
rc.getBoolean("pack", "cutdeltachains", getCutDeltaChains())); //$NON-NLS-1$ //$NON-NLS-2$
setBuildBitmaps(
rc.getBoolean("pack", "buildbitmaps", isBuildBitmaps())); //$NON-NLS-1$ //$NON-NLS-2$
setBitmapContiguousCommitCount(
rc.getInt("pack", "bitmapcontiguouscommitcount", //$NON-NLS-1$ //$NON-NLS-2$
getBitmapContiguousCommitCount()));
setBitmapRecentCommitCount(rc.getInt("pack", "bitmaprecentcommitcount", //$NON-NLS-1$ //$NON-NLS-2$
getBitmapRecentCommitCount()));
setBitmapRecentCommitSpan(rc.getInt("pack", "bitmaprecentcommitspan", //$NON-NLS-1$ //$NON-NLS-2$
getBitmapRecentCommitSpan()));
setBitmapDistantCommitSpan(rc.getInt("pack", "bitmapdistantcommitspan", //$NON-NLS-1$ //$NON-NLS-2$
getBitmapDistantCommitSpan()));
setBitmapExcessiveBranchCount(rc.getInt("pack", //$NON-NLS-1$
"bitmapexcessivebranchcount", getBitmapExcessiveBranchCount())); //$NON-NLS-1$
setBitmapInactiveBranchAgeInDays(
rc.getInt("pack", "bitmapinactivebranchageindays", //$NON-NLS-1$ //$NON-NLS-2$
getBitmapInactiveBranchAgeInDays()));
}
public String toString() {
final StringBuilder b = new StringBuilder();
b.append("maxDeltaDepth=").append(getMaxDeltaDepth()); //$NON-NLS-1$
b.append(", deltaSearchWindowSize=").append(getDeltaSearchWindowSize()); //$NON-NLS-1$
b.append(", deltaSearchMemoryLimit=").append(getDeltaSearchMemoryLimit()); //$NON-NLS-1$
b.append(", deltaSearchMemoryLimit=") //$NON-NLS-1$
.append(getDeltaSearchMemoryLimit());
b.append(", deltaCacheSize=").append(getDeltaCacheSize()); //$NON-NLS-1$
b.append(", deltaCacheLimit=").append(getDeltaCacheLimit()); //$NON-NLS-1$
b.append(", compressionLevel=").append(getCompressionLevel()); //$NON-NLS-1$
@ -735,6 +985,18 @@ public String toString() {
b.append(", reuseObjects=").append(isReuseObjects()); //$NON-NLS-1$
b.append(", deltaCompress=").append(isDeltaCompress()); //$NON-NLS-1$
b.append(", buildBitmaps=").append(isBuildBitmaps()); //$NON-NLS-1$
b.append(", bitmapContiguousCommitCount=") //$NON-NLS-1$
.append(getBitmapContiguousCommitCount());
b.append(", bitmapRecentCommitCount=") //$NON-NLS-1$
.append(getBitmapRecentCommitCount());
b.append(", bitmapRecentCommitSpan=") //$NON-NLS-1$
.append(getBitmapRecentCommitSpan());
b.append(", bitmapDistantCommitSpan=") //$NON-NLS-1$
.append(getBitmapDistantCommitSpan());
b.append(", bitmapExcessiveBranchCount=") //$NON-NLS-1$
.append(getBitmapExcessiveBranchCount());
b.append(", bitmapInactiveBranchAge=") //$NON-NLS-1$
.append(getBitmapInactiveBranchAgeInDays());
return b.toString();
}
}