From ce88e62edc4dfd6e07d43720c399ec169d594ad7 Mon Sep 17 00:00:00 2001 From: Anna Papitto Date: Tue, 2 May 2023 13:45:05 -0700 Subject: [PATCH] PackWriter: write the PackReverseIndex file PackWriter offers the ability to write out the pack file and its various index files, except for the newly introduced file-based reverse index. Now that PackReverseIndexWriter can write reverse index files, PackWriter#writeReverseIndex will write one for a pack if the corresponding config flag PackConfig#writeReverseIndex is on. Change-Id: Ib75dd2bbfb9ee9366d5aacb46700d8cf8af4823a Signed-off-by: Anna Papitto --- .../internal/storage/file/PackWriterTest.java | 33 +++++++++++++ .../storage/dfs/DfsGarbageCollector.java | 1 + .../internal/storage/pack/PackWriter.java | 33 +++++++++++++ .../org/eclipse/jgit/lib/ConfigConstants.java | 7 +++ .../eclipse/jgit/storage/pack/PackConfig.java | 46 +++++++++++++++++-- 5 files changed, 117 insertions(+), 3 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index 2a403c769..24a81b671 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -541,6 +541,39 @@ public void testWriteObjectSizeIndex_noDeltas() throws Exception { assertEquals(18787, objSizeIdx.getSize(idx.findPosition(knownBlob2))); } + @Test + public void testWriteReverseIndexConfig() { + assertFalse(config.isWriteReverseIndex()); + config.setWriteReverseIndex(true); + assertTrue(config.isWriteReverseIndex()); + } + + @Test + public void testWriteReverseIndexOff() throws Exception { + config.setWriteReverseIndex(false); + writer = new PackWriter(config, db.newObjectReader()); + ByteArrayOutputStream reverseIndexOutput = new ByteArrayOutputStream(); + + writer.writeReverseIndex(reverseIndexOutput); + + assertEquals(0, reverseIndexOutput.size()); + } + + @Test + public void testWriteReverseIndexOn() throws Exception { + config.setWriteReverseIndex(true); + writeVerifyPack4(false); + ByteArrayOutputStream reverseIndexOutput = new ByteArrayOutputStream(); + int headerBytes = 12; + int bodyBytes = 12; + int footerBytes = 40; + + writer.writeReverseIndex(reverseIndexOutput); + + assertTrue(reverseIndexOutput.size() == headerBytes + bodyBytes + + footerBytes); + } + @Test public void testExclude() throws Exception { // TestRepository closes repo diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 66bcf7398..92e23b8b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -577,6 +577,7 @@ private void packGarbage(ProgressMonitor pm) throws IOException { cfg.setReuseObjects(true); cfg.setDeltaCompress(false); cfg.setBuildBitmaps(false); + cfg.setWriteReverseIndex(false); try (PackWriter pw = new PackWriter(cfg, ctx); RevWalk pool = new RevWalk(ctx)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index bad572459..666e9d8cb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -62,6 +62,7 @@ import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; import org.eclipse.jgit.internal.storage.file.PackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; +import org.eclipse.jgit.internal.storage.file.PackReverseIndexWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectSizeQueue; import org.eclipse.jgit.lib.BatchingProgressMonitor; @@ -1136,6 +1137,38 @@ public void writeObjectSizeIndex(OutputStream objIdxStream) stats.timeWriting += System.currentTimeMillis() - writeStart; } + /** + * Whether the writer will write a reverse index file. The configuration + * flag must be on and the writer must be able to write corresponding + * forward index. + * + * @return whether the writer will write a reverse index file + */ + public boolean isReverseIndexEnabled() { + // Only write the reverse index if the writer is configured to and the + // forward index that it would correspond to will be written. + return config.isWriteReverseIndex() && !isIndexDisabled(); + } + + /** + * Write the pack's reverse index file to the output stream. + * + * @param stream + * where to write the file contents to + * @throws IOException + * if writing to the stream fails + */ + public void writeReverseIndex(OutputStream stream) throws IOException { + if (!isReverseIndexEnabled()) { + return; + } + long writeStart = System.currentTimeMillis(); + PackReverseIndexWriter writer = PackReverseIndexWriter + .createWriter(stream); + writer.write(sortByName(), packcsum); + stats.timeWriting += System.currentTimeMillis() - writeStart; + } + /** * Create a bitmap index file to match the pack file just written. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index c4b6bf955..4c080f476 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -744,6 +744,13 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT = "bitmaprecentcommitspan"; + /** + * The "pack.writeReverseIndex" key + * + * @since 6.6 + */ + public static final String CONFIG_KEY_WRITE_REVERSE_INDEX = "writeReverseIndex"; + /** * The "pack.buildBitmaps" key * @since 5.8 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index a0c978f6e..817742566 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -27,7 +27,10 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_COMPRESSION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DEPTH; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_INDEXVERSION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRESERVE_OLD_PACKS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRUNE_PRESERVED; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_DELTAS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_OBJECTS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT; @@ -36,10 +39,8 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WRITE_REVERSE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRESERVE_OLD_PACKS; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRUNE_PRESERVED; import java.time.Duration; import java.util.concurrent.Executor; @@ -161,6 +162,14 @@ public class PackConfig { */ public static final int DEFAULT_INDEX_VERSION = 2; + /** + * Default value of the write reverse index option: {@value} + * + * @see #setWriteReverseIndex(boolean) + * @since 6.6 + */ + public static final boolean DEFAULT_WRITE_REVERSE_INDEX = false; + /** * Default value of the build bitmaps option: {@value} * @@ -292,6 +301,8 @@ public class PackConfig { private int indexVersion = DEFAULT_INDEX_VERSION; + private boolean writeReverseIndex = DEFAULT_WRITE_REVERSE_INDEX; + private boolean buildBitmaps = DEFAULT_BUILD_BITMAPS; private int bitmapContiguousCommitCount = DEFAULT_BITMAP_CONTIGUOUS_COMMIT_COUNT; @@ -373,6 +384,7 @@ public PackConfig(PackConfig cfg) { this.threads = cfg.threads; this.executor = cfg.executor; this.indexVersion = cfg.indexVersion; + this.writeReverseIndex = cfg.writeReverseIndex; this.buildBitmaps = cfg.buildBitmaps; this.bitmapContiguousCommitCount = cfg.bitmapContiguousCommitCount; this.bitmapRecentCommitCount = cfg.bitmapRecentCommitCount; @@ -973,6 +985,31 @@ public void setIndexVersion(int version) { indexVersion = version; } + /** + * True if the writer should write reverse index files. + * + * Default setting: {@value #DEFAULT_WRITE_REVERSE_INDEX} + * + * @return whether the writer should write reverse index files + * @since 6.6 + */ + public boolean isWriteReverseIndex() { + return writeReverseIndex; + } + + /** + * Set whether the writer will write reverse index files. + * + * Default setting: {@value #DEFAULT_WRITE_REVERSE_INDEX} + * + * @param writeReverseIndex + * whether the writer should write reverse index files + * @since 6.6 + */ + public void setWriteReverseIndex(boolean writeReverseIndex) { + this.writeReverseIndex = writeReverseIndex; + } + /** * True if writer is allowed to build bitmaps for indexes. * @@ -1286,6 +1323,8 @@ public void fromConfig(Config rc) { setSinglePack(rc.getBoolean(CONFIG_PACK_SECTION, CONFIG_KEY_SINGLE_PACK, getSinglePack())); + setWriteReverseIndex(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_WRITE_REVERSE_INDEX, isWriteReverseIndex())); setBuildBitmaps(rc.getBoolean(CONFIG_PACK_SECTION, CONFIG_KEY_BUILD_BITMAPS, isBuildBitmaps())); setBitmapContiguousCommitCount(rc.getInt(CONFIG_PACK_SECTION, @@ -1347,6 +1386,7 @@ public String toString() { b.append(", reuseDeltas=").append(isReuseDeltas()); //$NON-NLS-1$ b.append(", reuseObjects=").append(isReuseObjects()); //$NON-NLS-1$ b.append(", deltaCompress=").append(isDeltaCompress()); //$NON-NLS-1$ + b.append(", writeReverseIndex=").append(isWriteReverseIndex()); // $NON-NLS-1$ b.append(", buildBitmaps=").append(isBuildBitmaps()); //$NON-NLS-1$ b.append(", bitmapContiguousCommitCount=") //$NON-NLS-1$ .append(getBitmapContiguousCommitCount());