From e55bad514bc7e66b5a2b3406e4f28545f437b8f0 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 1 Feb 2023 14:33:31 +0100 Subject: [PATCH 1/4] Document option "core.sha1Implementation" introduced in 59029aec Bug: 580310 Change-Id: I10f3d6f6b5af7ab96683994c9cbd85e6c18a5084 --- Documentation/config-options.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/config-options.md b/Documentation/config-options.md index b4a0c1d98..169a78657 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -42,6 +42,7 @@ For details on native git options see also the official [git config documentatio | `core.precomposeUnicode` | `true` on Mac OS | ✅ | MacOS only. When `true`, JGit reverts the unicode decomposition of filenames done by Mac OS. | | `core.quotePath` | `true` | ✅ | Commands that output paths (e.g. ls-files, diff), will quote "unusual" characters in the pathname by enclosing the pathname in double-quotes and escaping those characters with backslashes in the same way C escapes control characters (e.g. `\t` for TAB, `\n` for LF, `\\` for backslash) or bytes with values larger than `0x80` (e.g. octal `\302\265` for "micro" in UTF-8). | | `core.repositoryFormatVersion` | `1` | ⃞ | Internal version identifying the repository format and layout version. Don't set manually. | +| `core.sha1Implementation` | `java` | ⃞ | Choose the SHA1 implementation used by JGit. Set it to `java` to use JGit's Java implementation which detects SHA1 collisions if system property `org.eclipse.jgit.util.sha1.detectCollision` is unset or `true`. Set it to `jdkNative` to use the native implementation available in the JDK, can also be set using system property `org.eclipse.jgit.util.sha1.implementation`. If both are set the system property takes precedence. Performance of `jdkNative` is around 10% higher than `java` when `detectCollision=false` and 30% higher when `detectCollision=true`.| | `core.streamFileThreshold` | `50 MiB` | ⃞ | The size threshold beyond which objects must be streamed. | | `core.supportsAtomicFileCreation` | `true` | ⃞ | Whether the filesystem supports atomic file creation. | | `core.symlinks` | Auto detect if filesystem supports symlinks| ✅ | If false, symbolic links are checked out as small plain files that contain the link text. | From ed2cbd9e8a5714295d0651885ccf3f67088c59b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20=C5=BDivkov?= Date: Fri, 21 Oct 2022 16:32:03 +0200 Subject: [PATCH 2/4] Allow to perform PackedBatchRefUpdate without locking loose refs Add another newBatchUpdate method in the RefDirectory where we can control if the created PackedBatchRefUpdate will lock the loose refs or not. This can be useful in cases when we run programs which have exclusive access to a Git repository and we know that locking loose refs is unnecessary and just a performance loss. Change-Id: I7d0932eb1598a3871a2281b1a049021380234df9 (cherry picked from commit cb90ed08526bd51f04e5d72e3ba3cf5bd30c11e4) --- .../storage/file/PackedBatchRefUpdate.java | 12 +++++++++--- .../jgit/internal/storage/file/RefDirectory.java | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java index 9c1d33dc3..8b0ea4fcd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java @@ -86,10 +86,16 @@ */ class PackedBatchRefUpdate extends BatchRefUpdate { private RefDirectory refdb; + private boolean shouldLockLooseRefs; PackedBatchRefUpdate(RefDirectory refdb) { - super(refdb); - this.refdb = refdb; + this(refdb, true); + } + + PackedBatchRefUpdate(RefDirectory refdb, boolean shouldLockLooseRefs) { + super(refdb); + this.refdb = refdb; + this.shouldLockLooseRefs = shouldLockLooseRefs; } /** {@inheritDoc} */ @@ -155,7 +161,7 @@ public void execute(RevWalk walk, ProgressMonitor monitor, refdb.inProcessPackedRefsLock.lock(); try { PackedRefList oldPackedList; - if (!refdb.isInClone()) { + if (!refdb.isInClone() && shouldLockLooseRefs) { locks = lockLooseRefs(pending); if (locks == null) { return; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 07e38147f..b46ffe367 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -587,6 +587,21 @@ public PackedBatchRefUpdate newBatchUpdate() { return new PackedBatchRefUpdate(this); } + /** + * Create a new batch update to attempt on this database. + * + * @param shouldLockLooseRefs + * whether loose refs should be locked during the batch ref + * update. Note that this should only be set to {@code false} if + * the application using this ensures that no other ref updates + * run concurrently to avoid lost updates caused by a race. In + * such cases it can improve performance. + * @return a new batch update object + */ + public PackedBatchRefUpdate newBatchUpdate(boolean shouldLockLooseRefs) { + return new PackedBatchRefUpdate(this, shouldLockLooseRefs); + } + /** {@inheritDoc} */ @Override public boolean performsAtomicTransactions() { From 9424052f2797fabd8e1cee7f404eddadc7e3f72b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 10 Feb 2023 21:05:47 +0100 Subject: [PATCH 3/4] Add pack options to preserve and prune old pack files Add the options - pack.preserveOldPacks - pack.prunePreserved This allows to configure in git config if old packs should be preserved during gc and pruned during the next gc. The original implementation in 91132bb0 only allows to set these options using the API. Change-Id: I5b23ab4f317d12f5ccd234401419913e8263cc9a --- Documentation/config-options.md | 4 ++-- .../src/org/eclipse/jgit/pgm/Gc.java | 18 ++++++++++++------ .../org/eclipse/jgit/lib/ConfigConstants.java | 14 ++++++++++++++ .../eclipse/jgit/storage/pack/PackConfig.java | 6 ++++++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 169a78657..5d76483ac 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -100,8 +100,8 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.depth` | `50` | ✅ | Maximum depth of delta chain set up for the pack writer. | | `pack.indexVersion` | `2` | ✅ | Pack index file format version. | | `pack.minSizePreventRacyPack` | `100 MiB` | ⃞ | Minimum packfile size for which we wait before opening a newly written pack to prevent its lastModified timestamp could be racy if `pack.waitPreventRacyPack` is `true`. | -| `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs in a preserved directory. | -| `prunePreserved`, only via API of PackConfig | `false` | ⃞ | Whether to remove preserved pack files in a preserved directory. | +| `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs during gc in the `objects/pack/preserved` directory. This can avoid rare races between gc removing pack files and other concurrent operations. If this option is false data loss can occur in rare cases when an object is believed to be unreferenced when object repacking is running, and then garbage collection deletes it while another concurrent operation references this object shortly before garbage collection deletes it. When this happens, a new reference is created which points to a now missing object. | +| `pack.prunePreserved` | `false` | ⃞ | Whether to prune preserved pack files from the previous run of gc from the `objects/pack/preserved` directory. This helps to limit the additional storage space needed to preserve old packs when `pack.preserveOldPacks = true`. | | `pack.reuseDeltas` | `true` |⃞ | Whether to reuse deltas existing in repository. | | `pack.reuseObjects` | `true` | ⃞ | Whether to reuse existing objects representation in repository. | | `pack.searchForReuseTimeout` | | ⃞ | Search for reuse phase timeout. Expressed as a `Duration`, i.e.: `50sec`. | diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java index 177be7066..c87f0b6dc 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Gc.java @@ -10,6 +10,7 @@ package org.eclipse.jgit.pgm; +import org.eclipse.jgit.api.GarbageCollectCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.TextProgressMonitor; @@ -21,20 +22,25 @@ class Gc extends TextBuiltin { private boolean aggressive; @Option(name = "--preserve-oldpacks", usage = "usage_PreserveOldPacks") - private boolean preserveOldPacks; + private Boolean preserveOldPacks; @Option(name = "--prune-preserved", usage = "usage_PrunePreserved") - private boolean prunePreserved; + private Boolean prunePreserved; /** {@inheritDoc} */ @Override protected void run() { Git git = Git.wrap(db); try { - git.gc().setAggressive(aggressive) - .setPreserveOldPacks(preserveOldPacks) - .setPrunePreserved(prunePreserved) - .setProgressMonitor(new TextProgressMonitor(errw)).call(); + GarbageCollectCommand command = git.gc().setAggressive(aggressive) + .setProgressMonitor(new TextProgressMonitor(errw)); + if (preserveOldPacks != null) { + command.setPreserveOldPacks(preserveOldPacks.booleanValue()); + } + if (prunePreserved != null) { + command.setPrunePreserved(prunePreserved.booleanValue()); + } + command.call(); } catch (GitAPIException e) { throw die(e.getMessage(), e); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 6f76326bc..7d3430ffc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -771,4 +771,18 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT = "searchforreusetimeout"; + /** + * The "pack.preserveOldPacks" key + * + * @since 5.13.2 + */ + public static final String CONFIG_KEY_PRESERVE_OLD_PACKS = "preserveoldpacks"; + + /** + * The "pack.prunePreserved" key + * + * @since 5.13.2 + */ + public static final String CONFIG_KEY_PRUNE_PRESERVED = "prunepreserved"; + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index a10f6cf88..163e47588 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -37,6 +37,8 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRESERVE_OLD_PACKS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PRUNE_PRESERVED; import java.time.Duration; import java.util.concurrent.Executor; @@ -1267,6 +1269,10 @@ public void fromConfig(Config rc) { setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION, CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK, getMinSizePreventRacyPack())); + setPreserveOldPacks(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_PRESERVE_OLD_PACKS, DEFAULT_PRESERVE_OLD_PACKS)); + setPrunePreserved(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_PRUNE_PRESERVED, DEFAULT_PRUNE_PRESERVED)); } /** {@inheritDoc} */ From 012cb779304ccc6c089be06a2f5c12c293f11453 Mon Sep 17 00:00:00 2001 From: Prudhvi Akhil Alahari Date: Thu, 16 Feb 2023 16:41:55 +0530 Subject: [PATCH 4/4] Fix getPackedRefs to not throw NoSuchFileException Since Files.newInputStream is from java.nio package, it throws java.nio.file.NoSuchFileException. This was missed in the change I00da88e. Without this change, getPackedRefs fails with NoSuchFileException when there is no packed-refs file in a project. Change-Id: I93c202ddb73a0a5979af8e4d09e45f5645664b45 Signed-off-by: Prudhvi Akhil Alahari --- .../org/eclipse/jgit/internal/storage/file/RefDirectory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index d4ad190ff..c7322b17b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -35,6 +35,7 @@ import java.io.InterruptedIOException; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.security.DigestInputStream; import java.security.MessageDigest; @@ -911,7 +912,7 @@ PackedRefList getPackedRefs() throws IOException { try (InputStream stream = Files .newInputStream(packedRefsFile.toPath())) { // open the file to refresh attributes (on some NFS clients) - } catch (FileNotFoundException e) { + } catch (FileNotFoundException | NoSuchFileException e) { // Ignore as packed-refs may not exist } //$FALL-THROUGH$