From ad977f157242d4d6b5ea4c45b2aa0c15d20b58ae Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Tue, 20 Dec 2022 21:50:19 +0000 Subject: [PATCH] Allow the exclusions of refs prefixes from bitmap When running a GC.repack() against a repository with over one thousands of refs/heads and tens of millions of ObjectIds, the calculation of all bitmaps associated with all the refs would result in an unreasonable big file that would take up to several hours to compute. Test scenario: repo with 2500 heads / 10M obj Intel Xeon E5-2680 2.5GHz Before this change: 20 mins After this change and 2300 heads excluded: 10 mins (90s for bitmap) Having such a large bitmap file is also slow in the runtime processing and have negligible or even negative benefits, because the time lost in reading and decompressing the bitmap in memory would not be compensated by the time saved by using it. It is key to preserve the bitmaps for those refs that are mostly used in clone/fetch and give the ability to exlude some refs prefixes that are known to be less frequently accessed, even though they may actually be actively written. Example: Gerrit sandbox branches may even be actively used and selected automatically because its commits are very recent, however, they may bloat the bitmap, making it ineffective. A mono-repo with tens of thousands of developers may have a relatively small number of active branches where the CI/CD jobs are continuously fetching/cloning the code. However, because Gerrit allows the use of sandbox branches, the total number of refs/heads may be even tens to hundred thousands. Change-Id: I466dcde69fa008e7f7785735c977f6e150e3b644 Signed-off-by: Luca Milanesio --- Documentation/config-options.md | 1 + .../storage/pack/GcCommitSelectionTest.java | 18 +++++++++ .../jgit/internal/storage/file/GC.java | 23 +++++++++-- .../org/eclipse/jgit/lib/ConfigConstants.java | 6 +++ .../eclipse/jgit/storage/pack/PackConfig.java | 38 +++++++++++++++++++ 5 files changed, 82 insertions(+), 4 deletions(-) diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 19bcc3352..b4a0c1d98 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -86,6 +86,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.bitmapContiguousCommitCount` | `100` | ⃞ | Count of most recent commits for which to build bitmaps. | | `pack.bitmapDistantCommitSpan` | `5000` | ⃞ | Span of commits when building bitmaps for distant history. | | `pack.bitmapExcessiveBranchCount` | `100` | ⃞ | The count of branches deemed "excessive". If the count of branches in a repository exceeds this number and bitmaps are enabled, "inactive" branches will have fewer bitmaps than "active" branches. | +| `pack.bitmapExcludedRefsPrefixes` | | ⃞ | The refs prefixes to be excluded when building bitmaps. May be specified more than once to exclude multiple prefixes. | | `pack.bitmapInactiveBranchAgeInDays` | `90` | ⃞ | Age in days that marks a branch as "inactive" for bitmap creation. | | `pack.bitmapRecentCommitCount` | `20000` | ⃞ | Count at which to switch from `bitmapRecentCommitSpan` to `bitmapDistantCommitSpan`. | | `pack.bitmapRecentCommitSpan` | `100` | ⃞ | Span of commits when building bitmaps for recent history. | diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java index 55dfa697b..190ac8b64 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java @@ -220,6 +220,24 @@ public void testBitmapsForExcessiveBranches() throws Exception { gc.getStatistics().numberOfBitmaps); } + @Test + public void testBitmapsForExcludedBranches() throws Exception { + createNewCommitOnNewBranch("main"); + createNewCommitOnNewBranch("other"); + PackConfig packConfig = new PackConfig(); + packConfig.setBitmapExcludedRefsPrefixes(new String[] { "refs/heads/other" }); + gc.setPackConfig(packConfig); + gc.gc(); + assertEquals(1, + gc.getStatistics().numberOfBitmaps); + } + + private void createNewCommitOnNewBranch(String branchName) throws Exception { + BranchBuilder bb = tr.branch("refs/heads/" + branchName); + String msg = "New branch " + branchName; + bb.commit().message(msg).add("some-filename.txt", msg).create(); + } + @Test public void testSelectionOrderingWithChains() throws Exception { /*- diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index a14bb411f..9e9765949 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -796,6 +796,10 @@ public Collection repack() throws IOException { Set tagTargets = new HashSet<>(); Set indexObjects = listNonHEADIndexObjects(); + Set refsToExcludeFromBitmap = repo.getRefDatabase() + .getRefsByPrefix(pconfig.getBitmapExcludedRefsPrefixes()) + .stream().map(Ref::getObjectId).collect(Collectors.toSet()); + for (Ref ref : refsBefore) { checkCancelled(); nonHeads.addAll(listRefLogObjects(ref, 0)); @@ -840,7 +844,7 @@ public Collection repack() throws IOException { Pack heads = null; if (!allHeadsAndTags.isEmpty()) { heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, - tagTargets, excluded); + refsToExcludeFromBitmap, tagTargets, excluded); if (heads != null) { ret.add(heads); excluded.add(0, heads.getIndex()); @@ -848,13 +852,13 @@ public Collection repack() throws IOException { } if (!nonHeads.isEmpty()) { Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, - tagTargets, excluded); + PackWriter.NONE, tagTargets, excluded); if (rest != null) ret.add(rest); } if (!txnHeads.isEmpty()) { Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, - null, excluded); + PackWriter.NONE, null, excluded); if (txn != null) ret.add(txn); } @@ -1123,6 +1127,7 @@ private Set listNonHEADIndexObjects() private Pack writePack(@NonNull Set want, @NonNull Set have, @NonNull Set tags, + @NonNull Set excludedRefsTips, Set tagTargets, List excludeObjects) throws IOException { checkCancelled(); @@ -1154,7 +1159,8 @@ private Pack writePack(@NonNull Set want, if (excludeObjects != null) for (ObjectIdSet idx : excludeObjects) pw.excludeObjects(idx); - pw.preparePack(pm, want, have, PackWriter.NONE, tags); + pw.preparePack(pm, want, have, PackWriter.NONE, + union(tags, excludedRefsTips)); if (pw.getObjectCount() == 0) return null; checkCancelled(); @@ -1267,6 +1273,15 @@ private Pack writePack(@NonNull Set want, } } + private Set union(Set tags, + Set excludedRefsHeadsTips) { + HashSet unionSet = new HashSet<>( + tags.size() + excludedRefsHeadsTips.size()); + unionSet.addAll(tags); + unionSet.addAll(excludedRefsHeadsTips); + return unionSet; + } + private void checkCancelled() throws CancelledException { if (pm.isCancelled() || Thread.currentThread().isInterrupted()) { throw new CancelledException(JGitText.get().operationCanceled); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 924328d8a..6f76326bc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -620,6 +620,12 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT = "bitmapexcessivebranchcount"; + /** + * The "pack.bitmapExcludedRefsPrefixes" key + * @since 5.13.2 + */ + public static final String CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES = "bitmapexcludedrefsprefixes"; + /** * The "pack.bitmapInactiveBranchAgeInDays" key * @since 5.8 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 6aa8be642..a10f6cf88 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -16,6 +16,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_CONTIGUOUS_COMMIT_COUNT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BUILD_BITMAPS; @@ -225,6 +226,14 @@ public class PackConfig { */ public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90; + /** + * Default refs prefixes excluded from the calculation of pack bitmaps. + * + * @see #setBitmapExcludedRefsPrefixes(String[]) + * @since 5.13.2 + */ + public static final String[] DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES = new String[0]; + /** * Default max time to spend during the search for reuse phase. This * optimization is disabled by default: {@value} @@ -285,6 +294,8 @@ public class PackConfig { private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS; + private String[] bitmapExcludedRefsPrefixes = DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES; + private Duration searchForReuseTimeout = DEFAULT_SEARCH_FOR_REUSE_TIMEOUT; private boolean cutDeltaChains; @@ -1144,6 +1155,27 @@ public void setBitmapInactiveBranchAgeInDays(int ageInDays) { bitmapInactiveBranchAgeInDays = ageInDays; } + /** + * Get the refs prefixes excluded from the Bitmap. + * + * @return the refs prefixes excluded from the Bitmap. + * @since 5.13.2 + */ + public String[] getBitmapExcludedRefsPrefixes() { + return bitmapExcludedRefsPrefixes; + } + + /** + * Set the refs prefixes excluded from the Bitmap. + * + * @param excludedRefsPrefixes + * the refs prefixes excluded from the Bitmap. + * @since 5.13.2 + */ + public void setBitmapExcludedRefsPrefixes(String[] excludedRefsPrefixes) { + bitmapExcludedRefsPrefixes = excludedRefsPrefixes; + } + /** * Set the max time to spend during the search for reuse phase. * @@ -1220,6 +1252,12 @@ public void fromConfig(Config rc) { setBitmapInactiveBranchAgeInDays(rc.getInt(CONFIG_PACK_SECTION, CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS, getBitmapInactiveBranchAgeInDays())); + String[] excludedRefsPrefixesArray = rc.getStringList(CONFIG_PACK_SECTION, + null, + CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES); + if(excludedRefsPrefixesArray.length > 0) { + setBitmapExcludedRefsPrefixes(excludedRefsPrefixesArray); + } setSearchForReuseTimeout(Duration.ofSeconds(rc.getTimeUnit( CONFIG_PACK_SECTION, null, CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT,