From cd3fc7a2995c06cf2425f51758094e039c938559 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 18 Jan 2023 17:39:19 +0100 Subject: [PATCH] Speedup GC listing objects referenced from reflogs GC needs to get a ReflogReader for all existing refs to list all objects referenced from reflogs. The existing Repository#getReflogReader method accepts the ref name and then resolves the Ref to create a ReflogReader. GC calling that for a huge number of Refs one by one is very slow. GC first gets all Refs in bulk and then calls getReflogReader for each of them. Fix this by adding another getReflogReader method to Repository which accepts a Ref directly. This speeds up running JGit gc on a mirror clone of the Gerrit repository from 15:36 min to 1:08 min. The repository used in this test had 45k refs, 275k commits and 1.2m git objects. Change-Id: I474897fdc6652923e35d461c065a29f54d9949f4 --- .../.settings/.api_filters | 17 ---------- org.eclipse.jgit/.settings/.api_filters | 32 +++++++------------ .../internal/storage/file/FileRepository.java | 7 ++++ .../jgit/internal/storage/file/GC.java | 5 +-- .../src/org/eclipse/jgit/lib/Repository.java | 16 ++++++++++ 5 files changed, 36 insertions(+), 41 deletions(-) delete mode 100644 org.eclipse.jgit.http.server/.settings/.api_filters diff --git a/org.eclipse.jgit.http.server/.settings/.api_filters b/org.eclipse.jgit.http.server/.settings/.api_filters deleted file mode 100644 index 951a53bf3..000000000 --- a/org.eclipse.jgit.http.server/.settings/.api_filters +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 87deace39..27b322c91 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,5 +1,13 @@ + + + + + + + + @@ -8,11 +16,11 @@ - - + + - - + + @@ -24,22 +32,6 @@ - - - - - - - - - - - - - - - - diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index fecced1ae..90ee8def5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -31,6 +31,7 @@ import java.util.Objects; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; @@ -525,6 +526,12 @@ public ReflogReader getReflogReader(String refName) throws IOException { return new ReflogReaderImpl(this, ref.getName()); } + @Override + public @NonNull ReflogReader getReflogReader(@NonNull Ref ref) + throws IOException { + return new ReflogReaderImpl(this, ref.getName()); + } + /** {@inheritDoc} */ @Override public AttributesNodeProvider createAttributesNodeProvider() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 40c075ec5..a14bb411f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1019,10 +1019,7 @@ private void deleteTempPacksIdx() { * @throws IOException */ private Set listRefLogObjects(Ref ref, long minTime) throws IOException { - ReflogReader reflogReader = repo.getReflogReader(ref.getName()); - if (reflogReader == null) { - return Collections.emptySet(); - } + ReflogReader reflogReader = repo.getReflogReader(ref); List rlEntries = reflogReader .getReverseEntries(); if (rlEntries == null || rlEntries.isEmpty()) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 1e8a6c917..d3b3c6e8a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -1691,6 +1691,22 @@ public void setGitwebDescription(@Nullable String description) public abstract ReflogReader getReflogReader(String refName) throws IOException; + /** + * Get the reflog reader. Subclasses should override this method and provide + * a more efficient implementation. + * + * @param ref + * a Ref + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref, + * or {@code null} if the ref does not exist. + * @throws IOException + * @since 5.13.2 + */ + public @Nullable ReflogReader getReflogReader(@NonNull Ref ref) + throws IOException { + return getReflogReader(ref.getName()); + } + /** * Return the information stored in the file $GIT_DIR/MERGE_MSG. In this * file operations triggering a merge will store a template for the commit