From 91132bb05e767694d35230d515443d54ddc4eff6 Mon Sep 17 00:00:00 2001 From: James Melvin Date: Tue, 3 Jan 2017 11:41:56 -0700 Subject: [PATCH] gc: Add options to preserve and prune old pack files The new --preserve-oldpacks option moves old pack files into the preserved subdirectory instead of deleting them after repacking. The new --prune-preserved option prunes old pack files from the preserved subdirectory after repacking, but before potentially moving the latest old packfiles to this subdirectory. These options are designed to prevent stale file handle exceptions during git operations which can happen on users of NFS repos when repacking is done on them. The strategy is to preserve old pack files around until the next repack with the hopes that they will become unreferenced by then and not cause any exceptions to running processes when they are finally deleted (pruned). Change-Id: If3f729f0d9ce920ee2c3e6acdde46f2068be61d2 Signed-off-by: James Melvin --- .../jgit/pgm/internal/CLIText.properties | 2 + .../src/org/eclipse/jgit/pgm/Gc.java | 8 ++ .../storage/file/GcBasicPackingTest.java | 43 ++++++++++- .../jgit/api/GarbageCollectCommand.java | 32 ++++++++ .../jgit/internal/storage/file/GC.java | 52 +++++++++++-- .../storage/file/ObjectDirectory.java | 10 +++ .../eclipse/jgit/storage/pack/PackConfig.java | 75 +++++++++++++++++++ 7 files changed, 215 insertions(+), 7 deletions(-) diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index 1d4bf76b9..06e4d94f7 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -246,6 +246,8 @@ usage_LsTree=List the contents of a tree object usage_MakeCacheTree=Show the current cache tree structure usage_MergeBase=Find as good common ancestors as possible for a merge usage_MergesTwoDevelopmentHistories=Merges two development histories +usage_PreserveOldPacks=Preserve old pack files by moving them into the preserved subdirectory instead of deleting them after repacking +usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files usage_ReadDirCache= Read the DirCache 100 times usage_RebuildCommitGraph=Recreate a repository from another one's commit graph usage_RebuildRefTree=Copy references into a RefTree diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java index bf454760a..7289abb62 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java @@ -52,10 +52,18 @@ class Gc extends TextBuiltin { @Option(name = "--aggressive", usage = "usage_Aggressive") private boolean aggressive; + @Option(name = "--preserve-oldpacks", usage = "usage_PreserveOldPacks") + private boolean preserveOldPacks; + + @Option(name = "--prune-preserved", usage = "usage_PrunePreserved") + private boolean prunePreserved; + @Override protected void run() throws Exception { Git git = Git.wrap(db); git.gc().setAggressive(aggressive) + .setPreserveOldPacks(preserveOldPacks) + .setPrunePreserved(prunePreserved) .setProgressMonitor(new TextProgressMonitor(errw)).call(); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index 762feed3c..8a9ad8960 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -44,6 +44,8 @@ package org.eclipse.jgit.internal.storage.file; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -233,7 +235,45 @@ public void testDonePruneTooYoungPacks() throws Exception { } - private void configureGc(GC myGc, boolean aggressive) { + @Test + public void testPreserveAndPruneOldPacks() throws Exception { + testPreserveOldPacks(); + configureGc(gc, false).setPrunePreserved(true); + gc.gc(); + + assertFalse(repo.getObjectDatabase().getPreservedDirectory().exists()); + } + + private void testPreserveOldPacks() throws Exception { + BranchBuilder bb = tr.branch("refs/heads/master"); + bb.commit().message("P").add("P", "P").create(); + + // pack loose object into packfile + gc.setExpireAgeMillis(0); + gc.gc(); + File oldPackfile = tr.getRepository().getObjectDatabase().getPacks() + .iterator().next().getPackFile(); + assertTrue(oldPackfile.exists()); + + fsTick(); + bb.commit().message("B").add("B", "Q").create(); + + // repack again but now without a grace period for packfiles. We should + // end up with a new packfile and the old one should be placed in the + // preserved directory + gc.setPackExpireAgeMillis(0); + configureGc(gc, false).setPreserveOldPacks(true); + gc.gc(); + + File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); + String oldPackFileName = oldPackfile.getName(); + String oldPackName = oldPackFileName.substring(0, + oldPackFileName.lastIndexOf('.')) + ".old-pack"; //$NON-NLS-1$ + File preservePackFile = new File(oldPackDir, oldPackName); + assertTrue(preservePackFile.exists()); + } + + private PackConfig configureGc(GC myGc, boolean aggressive) { PackConfig pconfig = new PackConfig(repo); if (aggressive) { pconfig.setDeltaSearchWindowSize(250); @@ -242,5 +282,6 @@ private void configureGc(GC myGc, boolean aggressive) { } else pconfig = new PackConfig(repo); myGc.setPackConfig(pconfig); + return pconfig; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java index d0f729cc6..0f38db53b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java @@ -159,6 +159,38 @@ public GarbageCollectCommand setAggressive(boolean aggressive) { return this; } + /** + * Whether to preserve old pack files instead of deleting them. + * + * @since 4.7 + * @param preserveOldPacks + * whether to preserve old pack files + * @return this instance + */ + public GarbageCollectCommand setPreserveOldPacks(boolean preserveOldPacks) { + if (pconfig == null) + pconfig = new PackConfig(repo); + + pconfig.setPreserveOldPacks(preserveOldPacks); + return this; + } + + /** + * Whether to prune preserved pack files in the preserved directory. + * + * @since 4.7 + * @param prunePreserved + * whether to prune preserved pack files + * @return this instance + */ + public GarbageCollectCommand setPrunePreserved(boolean prunePreserved) { + if (pconfig == null) + pconfig = new PackConfig(repo); + + pconfig.setPrunePreserved(prunePreserved); + return this; + } + @Override public Properties call() throws GitAPIException { checkCallable(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index d67b9fa29..990b0be33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -211,9 +211,10 @@ public Collection gc() throws IOException, ParseException { /** * Delete old pack files. What is 'old' is defined by specifying a set of * old pack files and a set of new pack files. Each pack file contained in - * old pack files but not contained in new pack files will be deleted. If an - * expirationDate is set then pack files which are younger than the - * expirationDate will not be deleted. + * old pack files but not contained in new pack files will be deleted. If + * preserveOldPacks is set, keep a copy of the pack file in the preserve + * directory. If an expirationDate is set then pack files which are younger + * than the expirationDate will not be deleted nor preserved. * * @param oldPacks * @param newPacks @@ -222,6 +223,7 @@ public Collection gc() throws IOException, ParseException { */ private void deleteOldPacks(Collection oldPacks, Collection newPacks) throws ParseException, IOException { + prunePreserved(); long packExpireDate = getPackExpireDate(); oldPackLoop: for (PackFile oldPack : oldPacks) { String oldName = oldPack.getPackName(); @@ -238,11 +240,49 @@ private void deleteOldPacks(Collection oldPacks, prunePack(oldName); } } - // close the complete object database. Thats my only chance to force + // close the complete object database. That's my only chance to force // rescanning and to detect that certain pack files are now deleted. repo.getObjectDatabase().close(); } + /** + * Deletes old pack file, unless 'preserve-oldpacks' is set, in which case it + * moves the pack file to the preserved directory + * + * @param packFile + * @param packName + * @param ext + * @param deleteOptions + * @throws IOException + */ + private void removeOldPack(File packFile, String packName, PackExt ext, + int deleteOptions) throws IOException { + if (pconfig != null && pconfig.isPreserveOldPacks()) { + File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); + FileUtils.mkdir(oldPackDir, true); + + String oldPackName = "pack-" + packName + ".old-" + ext.getExtension(); //$NON-NLS-1$ //$NON-NLS-2$ + File oldPackFile = new File(oldPackDir, oldPackName); + FileUtils.rename(packFile, oldPackFile); + } else { + FileUtils.delete(packFile, deleteOptions); + } + } + + /** + * Delete the preserved directory including all pack files within + */ + private void prunePreserved() { + if (pconfig != null && pconfig.isPrunePreserved()) { + try { + FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(), + FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING); + } catch (IOException e) { + // Deletion of the preserved pack files failed. Silently return. + } + } + } + /** * Delete files associated with a single pack file. First try to delete the * ".pack" file because on some platforms the ".pack" file may be locked and @@ -262,7 +302,7 @@ private void prunePack(String packName) { for (PackExt ext : extensions) if (PackExt.PACK.equals(ext)) { File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - FileUtils.delete(f, deleteOptions); + removeOldPack(f, packName, ext, deleteOptions); break; } // The .pack file has been deleted. Delete as many as the other @@ -271,7 +311,7 @@ private void prunePack(String packName) { for (PackExt ext : extensions) { if (!PackExt.PACK.equals(ext)) { File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - FileUtils.delete(f, deleteOptions); + removeOldPack(f, packName, ext, deleteOptions); } } } catch (IOException e) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index c4db19a7e..eec7fb7c3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -125,6 +125,8 @@ public class ObjectDirectory extends FileObjectDatabase { private final File packDirectory; + private final File preservedDirectory; + private final File alternatesFile; private final AtomicReference packList; @@ -165,6 +167,7 @@ public ObjectDirectory(final Config cfg, final File dir, objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ packDirectory = new File(objects, "pack"); //$NON-NLS-1$ + preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(infoDirectory, "alternates"); //$NON-NLS-1$ packList = new AtomicReference(NO_PACKS); unpackedObjectCache = new UnpackedObjectCache(); @@ -189,6 +192,13 @@ public final File getDirectory() { return objects; } + /** + * @return the location of the preserved directory. + */ + public final File getPreservedDirectory() { + return preservedDirectory; + } + @Override public boolean exists() { return fs.exists(objects); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index d594e9767..fa18fc429 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -74,6 +74,20 @@ public class PackConfig { */ public static final boolean DEFAULT_REUSE_OBJECTS = true; + /** + * Default value of keep old packs option: {@value} + * + * @see #setPreserveOldPacks(boolean) + */ + public static final boolean DEFAULT_PRESERVE_OLD_PACKS = false; + + /** + * Default value of prune old packs option: {@value} + * + * @see #setPrunePreserved(boolean) + */ + public static final boolean DEFAULT_PRUNE_PRESERVED = false; + /** * Default value of delta compress option: {@value} * @@ -204,6 +218,10 @@ public class PackConfig { private boolean reuseObjects = DEFAULT_REUSE_OBJECTS; + private boolean preserveOldPacks = DEFAULT_PRESERVE_OLD_PACKS; + + private boolean prunePreserved = DEFAULT_PRUNE_PRESERVED; + private boolean deltaBaseAsOffset = DEFAULT_DELTA_BASE_AS_OFFSET; private boolean deltaCompress = DEFAULT_DELTA_COMPRESS; @@ -281,6 +299,8 @@ public PackConfig(PackConfig cfg) { this.compressionLevel = cfg.compressionLevel; this.reuseDeltas = cfg.reuseDeltas; this.reuseObjects = cfg.reuseObjects; + this.preserveOldPacks = cfg.preserveOldPacks; + this.prunePreserved = cfg.prunePreserved; this.deltaBaseAsOffset = cfg.deltaBaseAsOffset; this.deltaCompress = cfg.deltaCompress; this.maxDeltaDepth = cfg.maxDeltaDepth; @@ -363,6 +383,61 @@ public void setReuseObjects(boolean reuseObjects) { this.reuseObjects = reuseObjects; } + /** + * Checks whether to preserve old packs in a preserved directory + * + * Default setting: {@value #DEFAULT_PRESERVE_OLD_PACKS} + * + * @return true if repacking will preserve old pack files. + * @since 4.7 + */ + public boolean isPreserveOldPacks() { + return preserveOldPacks; + } + + /** + * Set preserve old packs configuration option for repacking. + * + * If enabled, old pack files are moved into a preserved subdirectory instead + * of being deleted + * + * Default setting: {@value #DEFAULT_PRESERVE_OLD_PACKS} + * + * @param preserveOldPacks + * boolean indicating whether or not preserve old pack files + * @since 4.7 + */ + public void setPreserveOldPacks(boolean preserveOldPacks) { + this.preserveOldPacks = preserveOldPacks; + } + + /** + * Checks whether to remove preserved pack files in a preserved directory + * + * Default setting: {@value #DEFAULT_PRUNE_PRESERVED} + * + * @return true if repacking will remove preserved pack files. + * @since 4.7 + */ + public boolean isPrunePreserved() { + return prunePreserved; + } + + /** + * Set prune preserved configuration option for repacking. + * + * If enabled, preserved pack files are removed from a preserved subdirectory + * + * Default setting: {@value #DEFAULT_PRESERVE_OLD_PACKS} + * + * @param prunePreserved + * boolean indicating whether or not preserve old pack files + * @since 4.7 + */ + public void setPrunePreserved(boolean prunePreserved) { + this.prunePreserved = prunePreserved; + } + /** * True if writer can use offsets to point to a delta base. *