From 2011fe06d2cbc15159342b263704f95a39f937c4 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Fri, 6 Jan 2023 14:14:48 -0700 Subject: [PATCH 01/13] FileSnapshotTest: Add more MISSING_FILE coverage Add a couple tests that confirm what the docs say about isModified() and equals(MISSING_FILE) behavior. Change-Id: I6093040ba3594934c3270331405a44b2634b97c5 Signed-off-by: Nasser Grainawi --- .../internal/storage/file/FileSnapshotTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 5e87b8f59..12773c240 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -209,6 +209,20 @@ public void fileSnapshotEquals() throws Exception { assertTrue(fs2.equals(fs1)); } + @Test + public void snapshotAndFileMissingIsNotModified() throws Exception { + File doesNotExist = trash.resolve("DOES_NOT_EXIST").toFile(); + FileSnapshot missing = FileSnapshot.save(doesNotExist); + assertFalse(missing.isModified(doesNotExist)); + } + + @Test + public void missingFileEquals() throws Exception { + FileSnapshot missing = FileSnapshot.save( + trash.resolve("DOES_NOT_EXIST").toFile()); + assertTrue(missing.equals(FileSnapshot.MISSING_FILE)); + } + @SuppressWarnings("boxing") @Test public void detectFileModified() throws IOException { From fed1a54935c0180e95804d42c1700b1f1819de3d Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Wed, 11 Jan 2023 21:25:30 -0800 Subject: [PATCH 02/13] Refresh 'objects' dir and retry if a loose object is not found A new loose object may not be immediately visible on a NFS client if it was created on another client. Refreshing the 'objects' dir and trying again can help work around the NFS behavior. Here's an E2E problem that this change can help fix. Consider a Gerrit multi-primary setup with repositories based on NFS. Add a new patch-set to an existing change and then immediately fetch the new patch-set of that change. If the fetch is handled by a Gerrit primary different that the one which created the patch-set, then we sometimes run into a MissingObjectException that causes the fetch to fail. Bug: 581317 Change-Id: Iccc6676c68ef13a1e8b2ff52b3eeca790a89a13d Signed-off-by: Kaushik Lingarkar --- Documentation/config-options.md | 2 +- .../internal/storage/file/LooseObjects.java | 71 +++++++++++++++++-- .../storage/file/ObjectDirectory.java | 2 +- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 9cb3c62a3..5b61df04b 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -45,7 +45,7 @@ For details on native git options see also the official [git config documentatio | `core.streamFileThreshold` | `50 MiB` | ⃞ | The size threshold beyond which objects must be streamed. | | `core.supportsAtomicFileCreation` | `true` | ⃞ | Whether the filesystem supports atomic file creation. | | `core.symlinks` | Auto detect if filesystem supports symlinks| ✅ | If false, symbolic links are checked out as small plain files that contain the link text. | -| `core.trustFolderStat` | `true` | ⃞ | Whether to trust the pack folder's and packed-refs file's file attributes (Java equivalent of stat command on *nix). When looking for pack files, if `false` JGit will always scan the `.git/objects/pack` folder and if set to `true` it assumes that pack files are unchanged if the file attributes of the pack folder are unchanged. When getting the list of packed refs, if `false` JGit will always read the packed-refs file and if set to `true` it uses the file attributes of the packed-refs file and will only read it if a file attribute has changed. Setting this option to `false` can help to workaround caching issues on NFS, but reduces performance.| +| `core.trustFolderStat` | `true` | ⃞ | Whether to trust the pack folder's, packed-refs file's and loose-objects folder's file attributes (Java equivalent of stat command on *nix). When looking for pack files, if `false` JGit will always scan the `.git/objects/pack` folder and if set to `true` it assumes that pack files are unchanged if the file attributes of the pack folder are unchanged. When getting the list of packed refs, if `false` JGit will always read the packed-refs file and if set to `true` it uses the file attributes of the packed-refs file and will only read it if a file attribute has changed. When looking for loose objects, if `false` and if a loose object is not found, JGit will open and close a stream to `.git/objects` folder (which can refresh its directory listing, at least on some NFS clients) and retry looking for that loose object. Setting this option to `false` can help to workaround caching issues on NFS, but reduces performance. | | `core.trustPackedRefsStat` | `unset` | ⃞ | Whether to trust the file attributes (Java equivalent of stat command on *nix) of the packed-refs file. If `never` JGit will ignore the file attributes of the packed-refs file and always read it. If `always` JGit will trust the file attributes of the packed-refs file and will only read it if a file attribute has changed. `after_open` behaves the same as `always`, except that the packed-refs file is opened and closed before its file attributes are considered. An open/close of the packed-refs file is known to refresh its file attributes, at least on some NFS clients. If `unset`, JGit will use the behavior described in `trustFolderStat`. | | `core.worktree` | Root directory of the working tree if it is not the parent directory of the `.git` directory | ✅ | The path to the root of the working tree. | diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java index 33621a1e9..b9af83d24 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -14,6 +14,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.StandardCopyOption; @@ -24,6 +25,8 @@ import org.eclipse.jgit.internal.storage.file.FileObjectDatabase.InsertLooseObjectResult; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -52,15 +55,22 @@ class LooseObjects { private final UnpackedObjectCache unpackedObjectCache; + private final boolean trustFolderStat; + /** * Initialize a reference to an on-disk object directory. * + * @param config + * configuration for the loose objects handler. * @param dir * the location of the objects directory. */ - LooseObjects(File dir) { + LooseObjects(Config config, File dir) { directory = dir; unpackedObjectCache = new UnpackedObjectCache(); + trustFolderStat = config.getBoolean( + ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); } /** @@ -98,6 +108,19 @@ boolean hasCached(AnyObjectId id) { * @return {@code true} if the specified object is stored as a loose object. */ boolean has(AnyObjectId objectId) { + boolean exists = hasWithoutRefresh(objectId); + if (trustFolderStat || exists) { + return exists; + } + try (InputStream stream = Files.newInputStream(directory.toPath())) { + // refresh directory to work around NFS caching issue + } catch (IOException e) { + return false; + } + return hasWithoutRefresh(objectId); + } + + private boolean hasWithoutRefresh(AnyObjectId objectId) { return fileFor(objectId).exists(); } @@ -183,6 +206,22 @@ ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { */ ObjectLoader getObjectLoader(WindowCursor curs, File path, AnyObjectId id) throws IOException { + try { + return getObjectLoaderWithoutRefresh(curs, path, id); + } catch (FileNotFoundException e) { + if (trustFolderStat) { + throw e; + } + try (InputStream stream = Files + .newInputStream(directory.toPath())) { + // refresh directory to work around NFS caching issues + } + return getObjectLoaderWithoutRefresh(curs, path, id); + } + } + + private ObjectLoader getObjectLoaderWithoutRefresh(WindowCursor curs, + File path, AnyObjectId id) throws IOException { try (FileInputStream in = new FileInputStream(path)) { unpackedObjectCache().add(id); return UnpackedObject.open(in, path, id, curs); @@ -203,16 +242,34 @@ UnpackedObjectCache unpackedObjectCache() { } long getSize(WindowCursor curs, AnyObjectId id) throws IOException { + try { + return getSizeWithoutRefresh(curs, id); + } catch (FileNotFoundException noFile) { + try { + if (trustFolderStat) { + throw noFile; + } + try (InputStream stream = Files + .newInputStream(directory.toPath())) { + // refresh directory to work around NFS caching issue + } + return getSizeWithoutRefresh(curs, id); + } catch (FileNotFoundException e) { + if (fileFor(id).exists()) { + throw noFile; + } + unpackedObjectCache().remove(id); + return -1; + } + } + } + + private long getSizeWithoutRefresh(WindowCursor curs, AnyObjectId id) + throws IOException { File f = fileFor(id); try (FileInputStream in = new FileInputStream(f)) { unpackedObjectCache().add(id); return UnpackedObject.getSize(in, id, curs); - } catch (FileNotFoundException noFile) { - if (f.exists()) { - throw noFile; - } - unpackedObjectCache().remove(id); - return -1; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 06c8cad3a..531fd78cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -120,7 +120,7 @@ public ObjectDirectory(final Config cfg, final File dir, File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ File preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); - loose = new LooseObjects(objects); + loose = new LooseObjects(config, objects); packed = new PackDirectory(config, packDirectory); preserved = new PackDirectory(config, preservedDirectory); this.fs = fs; From 21b2aef0aa9f71306733c7724f7ffff3e86206de Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Tue, 10 Jan 2023 16:15:42 -0700 Subject: [PATCH 03/13] Cache trustFolderStat/trustPackedRefsStat value per-instance Instead of re-reading the config every time the methods using these values were called, cache the config value at the time of instance construction. Caching the values improves performance for each of the method calls. These configs are set based on the filesystem storing the repository and unlikely to change while an application is running. Change-Id: I1cae26dad672dd28b766ac532a871671475652df Signed-off-by: Nasser Grainawi --- .../internal/storage/file/PackDirectory.java | 23 ++++++++----------- .../internal/storage/file/RefDirectory.java | 21 +++++++++-------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index f32909f44..a7f28c677 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -63,12 +63,12 @@ class PackDirectory { private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY, new Pack[0]); - private final Config config; - private final File directory; private final AtomicReference packList; + private final boolean trustFolderStat; + /** * Initialize a reference to an on-disk 'pack' directory. * @@ -78,9 +78,16 @@ class PackDirectory { * the location of the {@code pack} directory. */ PackDirectory(Config config, File directory) { - this.config = config; this.directory = directory; packList = new AtomicReference<>(NO_PACKS); + + // Whether to trust the pack folder's modification time. If set to false + // we will always scan the .git/objects/pack folder to check for new + // pack files. If set to true (default) we use the folder's size, + // modification time, and key (inode) and assume that no new pack files + // can be in this folder if these attributes have not changed. + trustFolderStat = config.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); } /** @@ -331,16 +338,6 @@ private boolean doLogExponentialBackoff(int n) { } boolean searchPacksAgain(PackList old) { - // Whether to trust the pack folder's modification time. If set - // to false we will always scan the .git/objects/pack folder to - // check for new pack files. If set to true (default) we use the - // lastmodified attribute of the folder and assume that no new - // pack files can be in this folder if his modification time has - // not changed. - boolean trustFolderStat = config.getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - return ((!trustFolderStat) || old.snapshot.isModified(directory)) && old != scanPacks(old); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 43ebce391..d4ad190ff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -179,6 +179,10 @@ public class RefDirectory extends RefDatabase { private List retrySleepMs = RETRY_SLEEP_MS; + private final boolean trustFolderStat; + + private final TrustPackedRefsStat trustPackedRefsStat; + RefDirectory(FileRepository db) { final FS fs = db.getFS(); parent = db; @@ -190,6 +194,13 @@ public class RefDirectory extends RefDatabase { looseRefs.set(RefList. emptyList()); packedRefs.set(NO_PACKED_REFS); + trustFolderStat = db.getConfig() + .getBoolean(ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + trustPackedRefsStat = db.getConfig() + .getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_TRUST_PACKED_REFS_STAT, + TrustPackedRefsStat.UNSET); } Repository getRepository() { @@ -891,16 +902,6 @@ else if (0 <= (idx = packed.find(dst.getName()))) } PackedRefList getPackedRefs() throws IOException { - boolean trustFolderStat = getRepository().getConfig().getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - TrustPackedRefsStat trustPackedRefsStat = - getRepository().getConfig().getEnum( - ConfigConstants.CONFIG_CORE_SECTION, - null, - ConfigConstants.CONFIG_KEY_TRUST_PACKED_REFS_STAT, - TrustPackedRefsStat.UNSET); - final PackedRefList curList = packedRefs.get(); switch (trustPackedRefsStat) { From 42d78f788c18423a1b01cbdcffcf58781ff8137e Mon Sep 17 00:00:00 2001 From: Michael Keppler Date: Fri, 6 Jan 2023 18:33:00 +0100 Subject: [PATCH 04/13] Upgrade maven plugins Remove tycho-extras-version, because Tycho and Tycho Extras are meanwhile in a single repository and maintained together. Update - build-helper-maven-plugin to 3.3.0 - eclipse-jarsigner-plugin to 1.3.5 - jacoco-maven-plugin to 0.8.8 - japicmp to 0.17.1 - maven-antrun-plugin to 3.1.0 - maven-clean-plugin to 3.2.0 - maven-compiler-plugin to 3.10.1 - maven-dependency-plugin to 3.5.0 - maven-deploy-plugin to 3.0.0 - maven-enforcer-plugin to 3.1.0 - maven-install-plugin to 3.1.0 - maven-jar-plugin to 3.3.0 - maven-javadoc-plugin to 3.4.1 - maven-jxr-plugin to 3.3.0 - maven-pmd-plugin to 3.20.0 - maven-project-info-reports-plugin to 3.4.2 - maven-resources-plugin to 3.3.0 - maven-shade-plugin to 3.4.1 - maven-site-plugin to 4.0.0-M4 - maven-surefire-plugin to 3.0.0-M8 - spotbugs-maven-plugin to 4.7.3.0 - spring-boot-maven-plugin to 2.7.7 Change-Id: I14d9ff06d2f509d782eb63adfa6b5733649f11f1 --- org.eclipse.jgit.benchmarks/pom.xml | 14 ++++----- org.eclipse.jgit.packaging/pom.xml | 1 - pom.xml | 45 ++++++++++++++--------------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/org.eclipse.jgit.benchmarks/pom.xml b/org.eclipse.jgit.benchmarks/pom.xml index 3e37e8263..f7ef238a6 100644 --- a/org.eclipse.jgit.benchmarks/pom.xml +++ b/org.eclipse.jgit.benchmarks/pom.xml @@ -56,7 +56,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + 3.1.0 enforce-maven @@ -76,7 +76,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.10.1 UTF-8 ${java.version} @@ -113,7 +113,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.4.1 package @@ -175,7 +175,7 @@ org.apache.maven.plugins maven-site-plugin - 3.9.1 + 4.0.0-M4 org.apache.maven.wagon @@ -187,17 +187,17 @@ org.apache.maven.plugins maven-surefire-report-plugin - 3.0.0-M5 + 3.0.0-M8 org.apache.maven.plugins maven-jxr-plugin - 3.1.1 + 3.3.0 org.apache.maven.plugins maven-project-info-reports-plugin - 3.1.1 + 3.4.2 diff --git a/org.eclipse.jgit.packaging/pom.xml b/org.eclipse.jgit.packaging/pom.xml index 06e968069..53ca7e3b9 100644 --- a/org.eclipse.jgit.packaging/pom.xml +++ b/org.eclipse.jgit.packaging/pom.xml @@ -24,7 +24,6 @@ 11 2.7.5 - ${tycho-version} jgit-4.17 diff --git a/pom.xml b/pom.xml index 729cd1390..f3afee264 100644 --- a/pom.xml +++ b/pom.xml @@ -163,20 +163,19 @@ 6.0.0 4.0.0 10.0.13 - 0.15.3 + 0.17.1 4.5.14 4.4.16 1.7.30 - 3.3.1 - 2.6.0 + 3.4.1 2.10 1.72 - 4.3.0 - 3.1.2 - 3.1.1 - 3.0.0-M5 + 4.7.3.0 + 3.4.2 + 3.3.0 + 3.0.0-M8 ${maven-surefire-plugin-version} - 3.8.1 + 3.10.1 2.12.1 2.2 3.20.2 @@ -223,7 +222,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.3.0 @@ -242,25 +241,25 @@ maven-clean-plugin - 3.1.0 + 3.2.0 org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.4.1 org.apache.maven.plugins maven-antrun-plugin - 3.0.0 + 3.1.0 org.apache.maven.plugins maven-dependency-plugin - 3.2.0 + 3.5.0 @@ -289,7 +288,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.2.0 + 3.3.0 @@ -312,7 +311,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.15.0 + 3.20.0 utf-8 100 @@ -335,17 +334,17 @@ org.eclipse.cbi.maven.plugins eclipse-jarsigner-plugin - 1.3.2 + 1.3.5 org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.8 org.apache.maven.plugins maven-site-plugin - 3.9.1 + 4.0.0-M4 org.apache.maven.wagon @@ -372,12 +371,12 @@ org.apache.maven.plugins maven-deploy-plugin - 3.0.0-M1 + 3.0.0 org.apache.maven.plugins maven-install-plugin - 3.0.0-M1 + 3.1.0 org.apache.maven.plugins @@ -387,12 +386,12 @@ org.apache.maven.plugins maven-resources-plugin - 3.2.0 + 3.3.0 org.springframework.boot spring-boot-maven-plugin - 2.5.4 + 2.7.7 org.eclipse.dash @@ -406,7 +405,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0 + 3.1.0 enforce-maven From b48f5739d7a88ff0b2a2bfd55a6e8ddb10b0520d Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 16 Jan 2023 14:27:32 +0100 Subject: [PATCH 05/13] Re-add servlet-api 4.0 to the target platform This was removed from the JGit target platform by mistake in 6ca3d219. Change-Id: Iedae0586fb96651255b67ed6dbb9ff7702c0ea54 --- .../org.eclipse.jgit.target/jgit-4.17.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.18.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.19.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.20.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.21.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.22.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.23.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.24.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.25.target | 7 ++++++- .../org.eclipse.jgit.target/jgit-4.26.target | 7 ++++++- .../org.eclipse.jgit.target/projects/jetty-10.0.x.tpd | 5 +++++ 11 files changed, 65 insertions(+), 10 deletions(-) diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target index 8f041a2d5..72f96f322 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target index 8f93f0fcd..a871c5809 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target index 88599aa12..819a06586 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20.target index 5a8dfae9e..cd615ff22 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.21.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.21.target index d5ccbc181..993c93f48 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.21.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.21.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.22.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.22.target index 04b6af503..86444a51a 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.22.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.22.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.23.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.23.target index a4a3c6473..2c1692f67 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.23.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.23.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.24.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.24.target index 4fed83f41..3c296b5c7 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.24.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.24.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.25.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.25.target index 1a7d8bb08..ad5d99516 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.25.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.25.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.26.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.26.target index fd1f0b0de..dce8c5581 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.26.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.26.target @@ -1,7 +1,7 @@ - + @@ -13,6 +13,11 @@ + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-10.0.x.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-10.0.x.tpd index 9fe89e2b3..e1afcff6b 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-10.0.x.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-10.0.x.tpd @@ -9,3 +9,8 @@ location jetty-10.0.x "https://download.eclipse.org/oomph/jetty/release/10.0.13/ org.eclipse.jetty.util [10.0.13,10.0.14] org.eclipse.jetty.util.ajax [10.0.13,10.0.14] } + +location jetty-10.0.6 "https://download.eclipse.org/eclipse/jetty/10.0.6/" { + jakarta.servlet-api [4.0.0, 5.0.0) + jakarta.servlet-api.source [4.0.0, 5.0.0) +} \ No newline at end of file From cd3fc7a2995c06cf2425f51758094e039c938559 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 18 Jan 2023 17:39:19 +0100 Subject: [PATCH 06/13] Speedup GC listing objects referenced from reflogs GC needs to get a ReflogReader for all existing refs to list all objects referenced from reflogs. The existing Repository#getReflogReader method accepts the ref name and then resolves the Ref to create a ReflogReader. GC calling that for a huge number of Refs one by one is very slow. GC first gets all Refs in bulk and then calls getReflogReader for each of them. Fix this by adding another getReflogReader method to Repository which accepts a Ref directly. This speeds up running JGit gc on a mirror clone of the Gerrit repository from 15:36 min to 1:08 min. The repository used in this test had 45k refs, 275k commits and 1.2m git objects. Change-Id: I474897fdc6652923e35d461c065a29f54d9949f4 --- .../.settings/.api_filters | 17 ---------- org.eclipse.jgit/.settings/.api_filters | 32 +++++++------------ .../internal/storage/file/FileRepository.java | 7 ++++ .../jgit/internal/storage/file/GC.java | 5 +-- .../src/org/eclipse/jgit/lib/Repository.java | 16 ++++++++++ 5 files changed, 36 insertions(+), 41 deletions(-) delete mode 100644 org.eclipse.jgit.http.server/.settings/.api_filters diff --git a/org.eclipse.jgit.http.server/.settings/.api_filters b/org.eclipse.jgit.http.server/.settings/.api_filters deleted file mode 100644 index 951a53bf3..000000000 --- a/org.eclipse.jgit.http.server/.settings/.api_filters +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 87deace39..27b322c91 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,5 +1,13 @@ + + + + + + + + @@ -8,11 +16,11 @@ - - + + - - + + @@ -24,22 +32,6 @@ - - - - - - - - - - - - - - - - diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index fecced1ae..90ee8def5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -31,6 +31,7 @@ import java.util.Objects; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; @@ -525,6 +526,12 @@ public ReflogReader getReflogReader(String refName) throws IOException { return new ReflogReaderImpl(this, ref.getName()); } + @Override + public @NonNull ReflogReader getReflogReader(@NonNull Ref ref) + throws IOException { + return new ReflogReaderImpl(this, ref.getName()); + } + /** {@inheritDoc} */ @Override public AttributesNodeProvider createAttributesNodeProvider() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 40c075ec5..a14bb411f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1019,10 +1019,7 @@ private void deleteTempPacksIdx() { * @throws IOException */ private Set listRefLogObjects(Ref ref, long minTime) throws IOException { - ReflogReader reflogReader = repo.getReflogReader(ref.getName()); - if (reflogReader == null) { - return Collections.emptySet(); - } + ReflogReader reflogReader = repo.getReflogReader(ref); List rlEntries = reflogReader .getReverseEntries(); if (rlEntries == null || rlEntries.isEmpty()) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 1e8a6c917..d3b3c6e8a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -1691,6 +1691,22 @@ public void setGitwebDescription(@Nullable String description) public abstract ReflogReader getReflogReader(String refName) throws IOException; + /** + * Get the reflog reader. Subclasses should override this method and provide + * a more efficient implementation. + * + * @param ref + * a Ref + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref, + * or {@code null} if the ref does not exist. + * @throws IOException + * @since 5.13.2 + */ + public @Nullable ReflogReader getReflogReader(@NonNull Ref ref) + throws IOException { + return getReflogReader(ref.getName()); + } + /** * Return the information stored in the file $GIT_DIR/MERGE_MSG. In this * file operations triggering a merge will store a template for the commit From eb3a708676e3487bbd97df384c09595034034d7e Mon Sep 17 00:00:00 2001 From: Harald Weiner Date: Mon, 24 Oct 2022 23:07:27 +0200 Subject: [PATCH 07/13] [pgm] Fetch-CLI: add support for shallow This adds support for shallow cloning. The CloneCommand and the FetchCommand now have the new options --depth, --shallow-since and --shallow-exclude to tell the server that the client doesn't want to download the complete history. Bug: https://bugs.eclipse.org/bugs/show_bug.cgi?id=475615 Change-Id: I8f113bed25dd6df64f2f95de6a59d4675ab8a903 --- .../tst/org/eclipse/jgit/pgm/CloneTest.java | 139 ++++++++++++++++++ .../jgit/pgm/internal/CLIText.properties | 7 + .../src/org/eclipse/jgit/pgm/Clone.java | 22 +++ .../src/org/eclipse/jgit/pgm/Fetch.java | 20 +++ .../eclipse/jgit/pgm/internal/CLIText.java | 1 + .../eclipse/jgit/pgm/opt/CmdLineParser.java | 2 + .../eclipse/jgit/pgm/opt/InstantHandler.java | 60 ++++++++ 7 files changed, 251 insertions(+) create mode 100644 org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java index 4cbd61c69..cbb5bbb9c 100644 --- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java @@ -17,14 +17,20 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.time.Instant; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.MockSystemReader; +import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.CLIRepositoryTestCase; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; @@ -41,10 +47,14 @@ public class CloneTest extends CLIRepositoryTestCase { private Git git; + private TestRepository tr; + @Override @Before public void setUp() throws Exception { super.setUp(); + + tr = new TestRepository<>(db); git = new Git(db); } @@ -112,6 +122,22 @@ private RevCommit createInitialCommit() throws Exception { return git.commit().setMessage("Initial commit").call(); } + private RevCommit createSecondCommit() throws Exception { + JGitTestUtil.writeTrashFile(db, "Test.txt", "Some change"); + git.add().addFilepattern("Test.txt").call(); + return git.commit() + .setCommitter(new PersonIdent(this.committer, tr.getDate())) + .setMessage("Second commit").call(); + } + + private RevCommit createThirdCommit() throws Exception { + JGitTestUtil.writeTrashFile(db, "change.txt", "another change"); + git.add().addFilepattern("change.txt").call(); + return git.commit() + .setCommitter(new PersonIdent(this.committer, tr.getDate())) + .setMessage("Third commit").call(); + } + @Test public void testCloneEmpty() throws Exception { File gitDir = db.getDirectory(); @@ -203,4 +229,117 @@ public void testCloneMirror() throws Exception { assertEquals("refs/*", fetchRefSpec.getDestination()); assertNotNull(git2.getRepository().exactRef("refs/meta/foo/bar")); } + + @Test + public void testDepth() throws Exception { + createInitialCommit(); + createSecondCommit(); + createThirdCommit(); + + File gitDir = db.getDirectory(); + String sourceURI = gitDir.toURI().toString(); + File target = createTempDirectory("target"); + String cmd = "git clone --depth 1 " + sourceURI + " " + + shellQuote(target.getPath()); + String[] result = execute(cmd); + assertArrayEquals(new String[] { + "Cloning into '" + target.getPath() + "'...", "", "" }, result); + + Git git2 = Git.open(target); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(1, log.size()); + RevCommit commit = log.get(0); + assertEquals(Set.of(commit.getId()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals("Third commit", commit.getFullMessage()); + assertEquals(0, commit.getParentCount()); + } + + @Test + public void testDepth2() throws Exception { + createInitialCommit(); + createSecondCommit(); + createThirdCommit(); + + File gitDir = db.getDirectory(); + String sourceURI = gitDir.toURI().toString(); + File target = createTempDirectory("target"); + String cmd = "git clone --depth 2 " + sourceURI + " " + + shellQuote(target.getPath()); + String[] result = execute(cmd); + assertArrayEquals(new String[] { + "Cloning into '" + target.getPath() + "'...", "", "" }, result); + + Git git2 = Git.open(target); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals(List.of("Third commit", "Second commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + } + + @Test + public void testCloneRepositoryWithShallowSince() throws Exception { + createInitialCommit(); + tr.tick(30); + RevCommit secondCommit = createSecondCommit(); + tr.tick(45); + createThirdCommit(); + + File gitDir = db.getDirectory(); + String sourceURI = gitDir.toURI().toString(); + File target = createTempDirectory("target"); + String cmd = "git clone --shallow-since=" + + Instant.ofEpochSecond(secondCommit.getCommitTime()).toString() + + " " + sourceURI + " " + shellQuote(target.getPath()); + String[] result = execute(cmd); + assertArrayEquals(new String[] { + "Cloning into '" + target.getPath() + "'...", "", "" }, result); + + Git git2 = Git.open(target); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals(List.of("Third commit", "Second commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + } + + @Test + public void testCloneRepositoryWithShallowExclude() throws Exception { + final RevCommit firstCommit = createInitialCommit(); + final RevCommit secondCommit = createSecondCommit(); + createThirdCommit(); + + File gitDir = db.getDirectory(); + String sourceURI = gitDir.toURI().toString(); + File target = createTempDirectory("target"); + String cmd = "git clone --shallow-exclude=" + + firstCommit.getId().getName() + " --shallow-exclude=" + + secondCommit.getId().getName() + " " + sourceURI + " " + + shellQuote(target.getPath()); + String[] result = execute(cmd); + assertArrayEquals(new String[] { + "Cloning into '" + target.getPath() + "'...", "", "" }, result); + + Git git2 = Git.open(target); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(1, log.size()); + assertEquals(List.of("Third commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + } + } diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index 48f4e857a..98d711d0f 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -137,6 +137,7 @@ metaVar_commitOrTag=COMMIT|TAG metaVar_commitPaths=paths metaVar_configFile=FILE metaVar_connProp=conn.prop +metaVar_depth= metaVar_diffAlg=ALGORITHM metaVar_directory=DIRECTORY metaVar_extraArgument=ours|theirs @@ -144,6 +145,7 @@ metaVar_file=FILE metaVar_filepattern=filepattern metaVar_gitDir=GIT_DIR metaVar_hostName=HOSTNAME +metaVar_instant= metaVar_lfsStorage=STORAGE metaVar_linesOfContext=lines metaVar_message=message @@ -168,6 +170,8 @@ metaVar_s3Region=REGION metaVar_s3StorageClass=STORAGE-CLASS metaVar_seconds=SECONDS metaVar_service=SERVICE +metaVar_shallowExclude= +metaVar_shallowSince= metaVar_tagLocalUser= metaVar_tool=TOOL metaVar_treeish=tree-ish @@ -374,6 +378,7 @@ usage_detectRenames=detect renamed files usage_diffAlgorithm=the diff algorithm to use. Currently supported are: 'myers', 'histogram' usage_DiffTool=git difftool is a Git command that allows you to compare and edit files between revisions using common diff tools.\ngit difftool is a frontend to git diff and accepts the same options and arguments. usage_MergeTool=git-mergetool - Run merge conflict resolution tools to resolve merge conflicts.\nUse git mergetool to run one of several merge utilities to resolve merge conflicts. It is typically run after git merge. +usage_depth=Limit fetching to the specified number of commits from the tip of each remote branch history. usage_directoriesToExport=directories to export usage_disableTheServiceInAllRepositories=disable the service in all repositories usage_displayAListOfAllRegisteredJgitCommands=Display a list of all registered jgit commands @@ -447,6 +452,8 @@ usage_resetMixed=Resets the index but not the working tree usage_runLfsStore=Run LFS Store in a given directory usage_S3NoSslVerify=Skip verification of Amazon server certificate and hostname usage_setTheGitRepositoryToOperateOn=set the git repository to operate on +usage_shallowExclude=Deepen or shorten the history of a shallow repository to exclude commits reachable from a specified remote branch or tag. +usage_shallowSince=Deepen or shorten the history of a shallow repository to include all reachable commits after . usage_show=Display one commit usage_showRefNamesMatchingCommits=Show ref names matching commits usage_showPatch=display patch diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java index f28915d3f..9f9fa8fe9 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java @@ -13,7 +13,10 @@ import java.io.File; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; @@ -48,6 +51,15 @@ class Clone extends AbstractFetchCommand implements CloneCommand.Callback { @Option(name = "--quiet", usage = "usage_quiet") private Boolean quiet; + @Option(name = "--depth", metaVar = "metaVar_depth", usage = "usage_depth") + private Integer depth = null; + + @Option(name = "--shallow-since", metaVar = "metaVar_shallowSince", usage = "usage_shallowSince") + private Instant shallowSince = null; + + @Option(name = "--shallow-exclude", metaVar = "metaVar_shallowExclude", usage = "usage_shallowExclude") + private List shallowExcludes = new ArrayList<>(); + @Option(name = "--recurse-submodules", usage = "usage_recurseSubmodules") private boolean cloneSubmodules; @@ -97,6 +109,16 @@ protected void run() throws Exception { .setMirror(isMirror).setNoCheckout(noCheckout).setBranch(branch) .setCloneSubmodules(cloneSubmodules).setTimeout(timeout); + if (depth != null) { + command.setDepth(depth.intValue()); + } + if (shallowSince != null) { + command.setShallowSince(shallowSince); + } + for (String shallowExclude : shallowExcludes) { + command.addShallowExclude(shallowExclude); + } + command.setGitDir(gitdir == null ? null : new File(gitdir)); command.setDirectory(localNameF); boolean msgs = quiet == null || !quiet.booleanValue(); diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java index fbce4a534..2e0c36b28 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java @@ -14,6 +14,8 @@ import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; +import java.util.ArrayList; import java.util.List; import org.eclipse.jgit.api.FetchCommand; @@ -62,6 +64,15 @@ void nothin(@SuppressWarnings("unused") final boolean ignored) { @Option(name = "--tags", usage="usage_tags", aliases = { "-t" }) private Boolean tags; + @Option(name = "--depth", metaVar = "metaVar_depth", usage = "usage_depth") + private Integer depth = null; + + @Option(name = "--shallow-since", metaVar = "metaVar_shallowSince", usage = "usage_shallowSince") + private Instant shallowSince = null; + + @Option(name = "--shallow-exclude", metaVar = "metaVar_shallowExclude", usage = "usage_shallowExclude") + private List shallowExcludes = new ArrayList<>(); + @Option(name = "--no-tags", usage = "usage_notags", aliases = { "-n" }) void notags(@SuppressWarnings("unused") final boolean ignored) { @@ -120,6 +131,15 @@ protected void run() { fetch.setTagOpt(tags.booleanValue() ? TagOpt.FETCH_TAGS : TagOpt.NO_TAGS); } + if (depth != null) { + fetch.setDepth(depth.intValue()); + } + if (shallowSince != null) { + fetch.setShallowSince(shallowSince); + } + for (String shallowExclude : shallowExcludes) { + fetch.addShallowExclude(shallowExclude); + } if (0 <= timeout) { fetch.setTimeout(timeout); } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java index 490f800c0..d07268b4c 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java @@ -214,6 +214,7 @@ public static String fatalError(String message) { /***/ public String metaVar_filepattern; /***/ public String metaVar_gitDir; /***/ public String metaVar_hostName; + /***/ public String metaVar_instant; /***/ public String metaVar_lfsStorage; /***/ public String metaVar_linesOfContext; /***/ public String metaVar_message; diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java index 5d32e6561..df0b39b52 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.io.Writer; import java.lang.reflect.Field; +import java.time.Instant; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -55,6 +56,7 @@ public class CmdLineParser extends org.kohsuke.args4j.CmdLineParser { registry.registerHandler(RevCommit.class, RevCommitHandler.class); registry.registerHandler(RevTree.class, RevTreeHandler.class); registry.registerHandler(List.class, OptionWithValuesListHandler.class); + registry.registerHandler(Instant.class, InstantHandler.class); } private final Repository db; diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java new file mode 100644 index 000000000..feee78e9b --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2022, Harald Weiner and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.pgm.opt; + +import java.time.Instant; + +import org.eclipse.jgit.pgm.internal.CLIText; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.OptionDef; +import org.kohsuke.args4j.spi.OptionHandler; +import org.kohsuke.args4j.spi.Parameters; +import org.kohsuke.args4j.spi.Setter; + +/** + * Custom argument handler {@link java.time.Instant} from string values. + *

+ * Assumes the parser has been initialized with a Repository. + * + * @since 6.5 + */ +public class InstantHandler extends OptionHandler { + /** + * Create a new handler for the command name. + *

+ * This constructor is used only by args4j. + * + * @param parser + * a {@link org.kohsuke.args4j.CmdLineParser} object. + * @param option + * a {@link org.kohsuke.args4j.OptionDef} object. + * @param setter + * a {@link org.kohsuke.args4j.spi.Setter} object. + */ + public InstantHandler(CmdLineParser parser, OptionDef option, + Setter setter) { + super(parser, option, setter); + } + + /** {@inheritDoc} */ + @Override + public int parseArguments(Parameters params) throws CmdLineException { + Instant instant = Instant.parse(params.getParameter(0)); + setter.addValue(instant); + return 1; + } + + /** {@inheritDoc} */ + @Override + public String getDefaultMetaVariable() { + return CLIText.get().metaVar_instant; + } +} From 611412a05528ba5898cf948f4d3959f1fbc4170a Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 16 Jan 2023 21:58:56 +0100 Subject: [PATCH 08/13] BatchingProgressMonitor: avoid int overflow when computing percentage When cloning huge repositories I observed percentage of object counts turning negative. This happened if lastWork * 100 exceeded Integer.MAX_VALUE. Change-Id: Ic5f5cf5a911a91338267aace4daba4b873ab3900 --- .../jgit/lib/TextProgressMonitorTest.java | 83 +++++++++++++++++++ .../jgit/lib/BatchingProgressMonitor.java | 6 +- 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TextProgressMonitorTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TextProgressMonitorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TextProgressMonitorTest.java new file mode 100644 index 000000000..55ca2cdea --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TextProgressMonitorTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2023, SAP SE or an SAP affiliate company and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TextProgressMonitorTest { + + private TextProgressMonitor m; + + private ByteArrayOutputStream buf; + + @Before + public void setup() { + buf = new ByteArrayOutputStream(); + m = new TextProgressMonitor( + new OutputStreamWriter(buf, StandardCharsets.UTF_8)); + } + + @Test + public void testSimple() throws Exception { + m.beginTask("task", 10); + for (int i = 0; i < 10; i++) { + m.update(1); + } + m.endTask(); + Assert.assertArrayEquals( + new String[] { "", "task: 10% ( 1/10)", + "task: 20% ( 2/10)", + "task: 30% ( 3/10)", + "task: 40% ( 4/10)", + "task: 50% ( 5/10)", + "task: 60% ( 6/10)", + "task: 70% ( 7/10)", + "task: 80% ( 8/10)", + "task: 90% ( 9/10)", + "task: 100% (10/10)", + "task: 100% (10/10)\n" }, + bufLines()); + } + + @Test + public void testLargeNumbers() throws Exception { + m.beginTask("task", 1_000_000_000); + for (int i = 0; i < 10; i++) { + m.update(100_000_000); + } + m.endTask(); + Assert.assertArrayEquals( + new String[] { "", + "task: 10% ( 100000000/1000000000)", + "task: 20% ( 200000000/1000000000)", + "task: 30% ( 300000000/1000000000)", + "task: 40% ( 400000000/1000000000)", + "task: 50% ( 500000000/1000000000)", + "task: 60% ( 600000000/1000000000)", + "task: 70% ( 700000000/1000000000)", + "task: 80% ( 800000000/1000000000)", + "task: 90% ( 900000000/1000000000)", + "task: 100% (1000000000/1000000000)", + "task: 100% (1000000000/1000000000)\n" }, + bufLines()); + } + + String[] bufLines() throws UnsupportedEncodingException { + String s = new String(buf.toString(StandardCharsets.UTF_8.name())); + return s.split("\r"); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchingProgressMonitor.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchingProgressMonitor.java index 2caefa4d9..49e295aed 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchingProgressMonitor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchingProgressMonitor.java @@ -176,7 +176,7 @@ void update(BatchingProgressMonitor pm, int completed) { } } else { // Display once per second or when 1% is done. - int currPercent = lastWork * 100 / totalWork; + int currPercent = Math.round(lastWork * 100F / totalWork); if (display) { pm.onUpdate(taskName, lastWork, totalWork, currPercent); output = true; @@ -201,8 +201,8 @@ void end(BatchingProgressMonitor pm) { if (totalWork == UNKNOWN) { pm.onEndTask(taskName, lastWork); } else { - int pDone = lastWork * 100 / totalWork; - pm.onEndTask(taskName, lastWork, totalWork, pDone); + int currPercent = Math.round(lastWork * 100F / totalWork); + pm.onEndTask(taskName, lastWork, totalWork, currPercent); } } if (timerFuture != null) From e4529cd39c42872e9b4f80d38659f9de37956634 Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Wed, 28 Dec 2022 01:09:52 +0000 Subject: [PATCH 09/13] PackWriterBitmapPreparer: do not include annotated tags in bitmap The annotated tags should be excluded from the bitmap associated with the heads-only packfile. However, this was not happening because of the check of exclusion of the peeled object instead of the objectId to be excluded from the bitmap. Sample use-case: refs/heads/main ^ | commit1 <-- commit2 <- annotated-tag1 <- tag1 ^ | commit0 When creating a bitmap for the above commit graph, before this change all the commits are included (3 bitmaps), which is incorrect, because all commits reachable from annotated tags should not be included. The heads-only bitmap should include only commit0 and commit1 but because PackWriterBitPreparer was checking for the peeled pointer of tag1 to be excluded (commit2) which was not found in the list of tags to exclude (annotated-tag1), the commit2 was included, even if it wasn't reachable only from the head. Add an additional check for exclusion of the original objectId for allowing the exclusion of annotated tags and their pointed commits. Add one specific test associated with an annotated tag for making sure that this use-case is covered also. Example repository benchmark for measuring the improvement: # refs: 400k (2k heads, 88k tags, 310k changes) # objects: 11M (88k of them are annotate tags) # packfiles: 2.7G Before this change: GC time: 5h clone --bare time: 7 mins After this change: GC time: 20 mins clone --bare time: 3 mins Bug: 581267 Signed-off-by: Luca Milanesio Change-Id: Iff2bfc6587153001837220189a120ead9ac649dc --- .../storage/pack/GcCommitSelectionTest.java | 34 +++++++++++++++++++ .../pack/PackWriterBitmapPreparer.java | 3 ++ 2 files changed, 37 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java index cc826c30b..55dfa697b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java @@ -82,6 +82,40 @@ private void testBitmapSpansNoMerges(boolean withTags) throws Exception { } } + @Test + public void testBitmapDoesNotIncludeAnnotatedTags() throws Exception { + /* + * Make sure that the bitmap generated for the following commit + * graph does not include commit2 because it is not reachable by any + * heads, despite being reachable from tag1 through the annotated-tag1. + * + * refs/heads/main + * ^ + * | + * commit1 <-- commit2 <- annotated-tag1 <- tag1 + * ^ + * | + * commit0 + */ + String mainBranch = "refs/heads/main"; + BranchBuilder bb = tr.branch(mainBranch); + + String commitMsg = "commit msg"; + String fileBody = "file body"; + String tagName = "tag1"; + bb.commit().message(commitMsg + " 1").add("file1", fileBody).create(); + RevCommit commit1 = bb.commit().message(commitMsg + " 2").add("file2", fileBody).create(); + RevCommit commit2 = bb.commit().message(commitMsg + " 3").add("file3", fileBody).create(); + tr.lightweightTag(tagName, tr.tag(tagName, commit2)); + tr.branch(mainBranch).update(commit1); + + gc.setExpireAgeMillis(0); + gc.gc(); + + // Create only 2 bitmaps, for commit0 and commit1, excluding commit2 + assertEquals(2, gc.getStatistics().numberOfBitmaps); + } + @Test public void testBitmapSpansWithMerges() throws Exception { /* diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index f1ede2acf..9f2b4d9c8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -408,6 +408,9 @@ private CommitSelectionHelper captureOldAndNewCommits(RevWalk rw, List newWantsByNewest = new ArrayList<>(want.size()); Set newWants = new HashSet<>(want.size()); for (AnyObjectId objectId : want) { + if(excludeFromBitmapSelection.contains(objectId)) { + continue; + } RevObject ro = rw.peel(rw.parseAny(objectId)); if (!(ro instanceof RevCommit) || reuse.contains(ro) || excludeFromBitmapSelection.contains(ro)) { From ad977f157242d4d6b5ea4c45b2aa0c15d20b58ae Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Tue, 20 Dec 2022 21:50:19 +0000 Subject: [PATCH 10/13] Allow the exclusions of refs prefixes from bitmap When running a GC.repack() against a repository with over one thousands of refs/heads and tens of millions of ObjectIds, the calculation of all bitmaps associated with all the refs would result in an unreasonable big file that would take up to several hours to compute. Test scenario: repo with 2500 heads / 10M obj Intel Xeon E5-2680 2.5GHz Before this change: 20 mins After this change and 2300 heads excluded: 10 mins (90s for bitmap) Having such a large bitmap file is also slow in the runtime processing and have negligible or even negative benefits, because the time lost in reading and decompressing the bitmap in memory would not be compensated by the time saved by using it. It is key to preserve the bitmaps for those refs that are mostly used in clone/fetch and give the ability to exlude some refs prefixes that are known to be less frequently accessed, even though they may actually be actively written. Example: Gerrit sandbox branches may even be actively used and selected automatically because its commits are very recent, however, they may bloat the bitmap, making it ineffective. A mono-repo with tens of thousands of developers may have a relatively small number of active branches where the CI/CD jobs are continuously fetching/cloning the code. However, because Gerrit allows the use of sandbox branches, the total number of refs/heads may be even tens to hundred thousands. Change-Id: I466dcde69fa008e7f7785735c977f6e150e3b644 Signed-off-by: Luca Milanesio --- Documentation/config-options.md | 1 + .../storage/pack/GcCommitSelectionTest.java | 18 +++++++++ .../jgit/internal/storage/file/GC.java | 23 +++++++++-- .../org/eclipse/jgit/lib/ConfigConstants.java | 6 +++ .../eclipse/jgit/storage/pack/PackConfig.java | 38 +++++++++++++++++++ 5 files changed, 82 insertions(+), 4 deletions(-) diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 19bcc3352..b4a0c1d98 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -86,6 +86,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.bitmapContiguousCommitCount` | `100` | ⃞ | Count of most recent commits for which to build bitmaps. | | `pack.bitmapDistantCommitSpan` | `5000` | ⃞ | Span of commits when building bitmaps for distant history. | | `pack.bitmapExcessiveBranchCount` | `100` | ⃞ | The count of branches deemed "excessive". If the count of branches in a repository exceeds this number and bitmaps are enabled, "inactive" branches will have fewer bitmaps than "active" branches. | +| `pack.bitmapExcludedRefsPrefixes` | | ⃞ | The refs prefixes to be excluded when building bitmaps. May be specified more than once to exclude multiple prefixes. | | `pack.bitmapInactiveBranchAgeInDays` | `90` | ⃞ | Age in days that marks a branch as "inactive" for bitmap creation. | | `pack.bitmapRecentCommitCount` | `20000` | ⃞ | Count at which to switch from `bitmapRecentCommitSpan` to `bitmapDistantCommitSpan`. | | `pack.bitmapRecentCommitSpan` | `100` | ⃞ | Span of commits when building bitmaps for recent history. | diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java index 55dfa697b..190ac8b64 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java @@ -220,6 +220,24 @@ public void testBitmapsForExcessiveBranches() throws Exception { gc.getStatistics().numberOfBitmaps); } + @Test + public void testBitmapsForExcludedBranches() throws Exception { + createNewCommitOnNewBranch("main"); + createNewCommitOnNewBranch("other"); + PackConfig packConfig = new PackConfig(); + packConfig.setBitmapExcludedRefsPrefixes(new String[] { "refs/heads/other" }); + gc.setPackConfig(packConfig); + gc.gc(); + assertEquals(1, + gc.getStatistics().numberOfBitmaps); + } + + private void createNewCommitOnNewBranch(String branchName) throws Exception { + BranchBuilder bb = tr.branch("refs/heads/" + branchName); + String msg = "New branch " + branchName; + bb.commit().message(msg).add("some-filename.txt", msg).create(); + } + @Test public void testSelectionOrderingWithChains() throws Exception { /*- diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index a14bb411f..9e9765949 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -796,6 +796,10 @@ public Collection repack() throws IOException { Set tagTargets = new HashSet<>(); Set indexObjects = listNonHEADIndexObjects(); + Set refsToExcludeFromBitmap = repo.getRefDatabase() + .getRefsByPrefix(pconfig.getBitmapExcludedRefsPrefixes()) + .stream().map(Ref::getObjectId).collect(Collectors.toSet()); + for (Ref ref : refsBefore) { checkCancelled(); nonHeads.addAll(listRefLogObjects(ref, 0)); @@ -840,7 +844,7 @@ public Collection repack() throws IOException { Pack heads = null; if (!allHeadsAndTags.isEmpty()) { heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, - tagTargets, excluded); + refsToExcludeFromBitmap, tagTargets, excluded); if (heads != null) { ret.add(heads); excluded.add(0, heads.getIndex()); @@ -848,13 +852,13 @@ public Collection repack() throws IOException { } if (!nonHeads.isEmpty()) { Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, - tagTargets, excluded); + PackWriter.NONE, tagTargets, excluded); if (rest != null) ret.add(rest); } if (!txnHeads.isEmpty()) { Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, - null, excluded); + PackWriter.NONE, null, excluded); if (txn != null) ret.add(txn); } @@ -1123,6 +1127,7 @@ private Set listNonHEADIndexObjects() private Pack writePack(@NonNull Set want, @NonNull Set have, @NonNull Set tags, + @NonNull Set excludedRefsTips, Set tagTargets, List excludeObjects) throws IOException { checkCancelled(); @@ -1154,7 +1159,8 @@ private Pack writePack(@NonNull Set want, if (excludeObjects != null) for (ObjectIdSet idx : excludeObjects) pw.excludeObjects(idx); - pw.preparePack(pm, want, have, PackWriter.NONE, tags); + pw.preparePack(pm, want, have, PackWriter.NONE, + union(tags, excludedRefsTips)); if (pw.getObjectCount() == 0) return null; checkCancelled(); @@ -1267,6 +1273,15 @@ private Pack writePack(@NonNull Set want, } } + private Set union(Set tags, + Set excludedRefsHeadsTips) { + HashSet unionSet = new HashSet<>( + tags.size() + excludedRefsHeadsTips.size()); + unionSet.addAll(tags); + unionSet.addAll(excludedRefsHeadsTips); + return unionSet; + } + private void checkCancelled() throws CancelledException { if (pm.isCancelled() || Thread.currentThread().isInterrupted()) { throw new CancelledException(JGitText.get().operationCanceled); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 924328d8a..6f76326bc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -620,6 +620,12 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT = "bitmapexcessivebranchcount"; + /** + * The "pack.bitmapExcludedRefsPrefixes" key + * @since 5.13.2 + */ + public static final String CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES = "bitmapexcludedrefsprefixes"; + /** * The "pack.bitmapInactiveBranchAgeInDays" key * @since 5.8 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 6aa8be642..a10f6cf88 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -16,6 +16,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_CONTIGUOUS_COMMIT_COUNT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BUILD_BITMAPS; @@ -225,6 +226,14 @@ public class PackConfig { */ public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90; + /** + * Default refs prefixes excluded from the calculation of pack bitmaps. + * + * @see #setBitmapExcludedRefsPrefixes(String[]) + * @since 5.13.2 + */ + public static final String[] DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES = new String[0]; + /** * Default max time to spend during the search for reuse phase. This * optimization is disabled by default: {@value} @@ -285,6 +294,8 @@ public class PackConfig { private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS; + private String[] bitmapExcludedRefsPrefixes = DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES; + private Duration searchForReuseTimeout = DEFAULT_SEARCH_FOR_REUSE_TIMEOUT; private boolean cutDeltaChains; @@ -1144,6 +1155,27 @@ public void setBitmapInactiveBranchAgeInDays(int ageInDays) { bitmapInactiveBranchAgeInDays = ageInDays; } + /** + * Get the refs prefixes excluded from the Bitmap. + * + * @return the refs prefixes excluded from the Bitmap. + * @since 5.13.2 + */ + public String[] getBitmapExcludedRefsPrefixes() { + return bitmapExcludedRefsPrefixes; + } + + /** + * Set the refs prefixes excluded from the Bitmap. + * + * @param excludedRefsPrefixes + * the refs prefixes excluded from the Bitmap. + * @since 5.13.2 + */ + public void setBitmapExcludedRefsPrefixes(String[] excludedRefsPrefixes) { + bitmapExcludedRefsPrefixes = excludedRefsPrefixes; + } + /** * Set the max time to spend during the search for reuse phase. * @@ -1220,6 +1252,12 @@ public void fromConfig(Config rc) { setBitmapInactiveBranchAgeInDays(rc.getInt(CONFIG_PACK_SECTION, CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS, getBitmapInactiveBranchAgeInDays())); + String[] excludedRefsPrefixesArray = rc.getStringList(CONFIG_PACK_SECTION, + null, + CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES); + if(excludedRefsPrefixesArray.length > 0) { + setBitmapExcludedRefsPrefixes(excludedRefsPrefixesArray); + } setSearchForReuseTimeout(Duration.ofSeconds(rc.getTimeUnit( CONFIG_PACK_SECTION, null, CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT, From 8040936f8aee4f0637339ab7c1dcfc56fe612cff Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 31 Jan 2023 23:45:07 +0100 Subject: [PATCH 11/13] Silence API warnings introduced by I466dcde6 Change-Id: I510510da34d33757c2f83af8cd1e26f6206a486a --- .../.settings/.api_filters | 17 +++++++ org.eclipse.jgit/.settings/.api_filters | 48 +++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 org.eclipse.jgit.http.server/.settings/.api_filters diff --git a/org.eclipse.jgit.http.server/.settings/.api_filters b/org.eclipse.jgit.http.server/.settings/.api_filters new file mode 100644 index 000000000..951a53bf3 --- /dev/null +++ b/org.eclipse.jgit.http.server/.settings/.api_filters @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 27b322c91..cc4916cf8 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,14 +1,12 @@ - - + + - + - - @@ -24,6 +22,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -32,6 +62,14 @@ + + + + + + + + From 76508320025a2258ccf169de2f559ad01f27281b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 4 Oct 2022 15:42:25 +0200 Subject: [PATCH 12/13] FetchCommand: fix fetchSubmodules to work on a Ref to a blob FetchCommand#fetchSubmodules assumed that FETCH_HEAD can always be parsed as a tree. This isn't true if it refers to a Ref referring to a BLOB. This is e.g. used in Gerrit for Refs like refs/sequences/changes which are used to implement sequences stored in git. Change-Id: I414f5b7d9f2184b2d7d53af1dfcd68cccb725ca4 --- org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java index 90c1515b0..7290d83df 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java @@ -140,6 +140,9 @@ private void fetchSubmodules(FetchResult results) if (fetchHead == null) { return; } + if (revWalk.parseAny(fetchHead).getType() == Constants.OBJ_BLOB) { + return; + } walk.setTree(revWalk.parseTree(fetchHead)); while (walk.next()) { try (Repository submoduleRepo = walk.getRepository()) { From 21e902dd7fa4ff53dc35fd7c48f8b5edc52f8eea Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Wed, 18 May 2022 13:31:30 +0100 Subject: [PATCH 13/13] Shortcut during git fetch for avoiding looping through all local refs The FetchProcess needs to verify that all the refs received point to objects that are reachable from the local refs, which could be very expensive but is needed to avoid missing objects exceptions because of broken chains. When the local repository has a lot of refs (e.g. millions) and the client is fetching a non-commit object (e.g. refs/sequences/changes in Gerrit) the reachability check on all local refs can be very expensive compared to the time to fetch the remote ref. Example for a 2M refs repository: - fetching a single non-commit object: 50ms - checking the reachability of local refs: 30s A ref pointing to a non-commit object doesn't have any parent or successor objects, hence would never need to have a reachability check done. Skipping the askForIsComplete() altogether would save the 30s time spent in an unnecessary phase. Signed-off-by: Luca Milanesio Change-Id: I09ac66ded45cede199ba30f9e71cc1055f00941b --- .../eclipse/jgit/transport/FetchProcess.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java index 2cedd4b07..507795af5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java @@ -47,6 +47,7 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.util.StringUtils; @@ -378,11 +379,19 @@ private void updateFETCH_HEAD(FetchResult result) throws IOException { private boolean askForIsComplete() throws TransportException { try { try (ObjectWalk ow = new ObjectWalk(transport.local)) { - for (ObjectId want : askFor.keySet()) - ow.markStart(ow.parseAny(want)); - for (Ref ref : localRefs().values()) - ow.markUninteresting(ow.parseAny(ref.getObjectId())); - ow.checkConnectivity(); + boolean hasCommitObject = false; + for (ObjectId want : askFor.keySet()) { + RevObject obj = ow.parseAny(want); + ow.markStart(obj); + hasCommitObject |= obj.getType() == Constants.OBJ_COMMIT; + } + // Checking connectivity makes sense on commits only + if (hasCommitObject) { + for (Ref ref : localRefs().values()) { + ow.markUninteresting(ow.parseAny(ref.getObjectId())); + } + ow.checkConnectivity(); + } } return true; } catch (MissingObjectException e) {