matches, AbbreviatedObjectId id,
}
ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException {
- File path = fileFor(id);
- try (FileInputStream in = new FileInputStream(path)) {
- unpackedObjectCache.add(id);
- return UnpackedObject.open(in, path, id, curs);
- } catch (FileNotFoundException noFile) {
- if (path.exists()) {
- throw noFile;
+ int readAttempts = 0;
+ while (readAttempts < MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS) {
+ readAttempts++;
+ File path = fileFor(id);
+ try {
+ return getObjectLoader(curs, path, id);
+ } catch (FileNotFoundException noFile) {
+ if (path.exists()) {
+ throw noFile;
+ }
+ break;
+ } catch (IOException e) {
+ if (!FileUtils.isStaleFileHandleInCausalChain(e)) {
+ throw e;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(MessageFormat.format(
+ JGitText.get().looseObjectHandleIsStale, id.name(),
+ Integer.valueOf(readAttempts), Integer.valueOf(
+ MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS)));
+ }
}
- unpackedObjectCache.remove(id);
- return null;
}
+ unpackedObjectCache().remove(id);
+ return null;
+ }
+
+ /**
+ * Provides a loader for an objectId
+ *
+ * @param curs
+ * cursor on the database
+ * @param path
+ * the path of the loose object
+ * @param id
+ * the object id
+ * @return a loader for the loose file object
+ * @throws IOException
+ * when file does not exist or it could not be opened
+ */
+ ObjectLoader getObjectLoader(WindowCursor curs, File path, AnyObjectId id)
+ throws IOException {
+ try (FileInputStream in = new FileInputStream(path)) {
+ unpackedObjectCache().add(id);
+ return UnpackedObject.open(in, path, id, curs);
+ }
+ }
+
+ /**
+ *
+ * Getter for the field unpackedObjectCache.
+ *
+ * This accessor is particularly useful to allow mocking of this class for
+ * testing purposes.
+ *
+ * @return the cache of the objects currently unpacked.
+ */
+ UnpackedObjectCache unpackedObjectCache() {
+ return unpackedObjectCache;
}
long getSize(WindowCursor curs, AnyObjectId id) throws IOException {
File f = fileFor(id);
try (FileInputStream in = new FileInputStream(f)) {
- unpackedObjectCache.add(id);
+ unpackedObjectCache().add(id);
return UnpackedObject.getSize(in, id, curs);
} catch (FileNotFoundException noFile) {
if (f.exists()) {
throw noFile;
}
- unpackedObjectCache.remove(id);
+ unpackedObjectCache().remove(id);
return -1;
}
}
@@ -207,7 +263,7 @@ private InsertLooseObjectResult tryMove(File tmp, File dst, ObjectId id)
Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst),
StandardCopyOption.ATOMIC_MOVE);
dst.setReadOnly();
- unpackedObjectCache.add(id);
+ unpackedObjectCache().add(id);
return InsertLooseObjectResult.INSERTED;
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java
index 73745d8c6..f32909f44 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java
@@ -33,6 +33,7 @@
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.PackInvalidException;
import org.eclipse.jgit.errors.PackMismatchException;
+import org.eclipse.jgit.errors.SearchForReuseTimeout;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
import org.eclipse.jgit.internal.storage.pack.PackExt;
@@ -264,7 +265,10 @@ void selectRepresentation(PackWriter packer, ObjectToPack otp,
p.resetTransientErrorCount();
if (rep != null) {
packer.select(otp, rep);
+ packer.checkSearchForReuseTimeout();
}
+ } catch (SearchForReuseTimeout e) {
+ break SEARCH;
} catch (PackMismatchException e) {
// Pack was modified; refresh the entire pack list.
//
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
index 3e4b5df6a..61f92d2e1 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
@@ -25,6 +25,7 @@
import java.lang.ref.WeakReference;
import java.security.MessageDigest;
import java.text.MessageFormat;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -54,6 +55,7 @@
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.errors.SearchForReuseTimeout;
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder;
@@ -262,6 +264,12 @@ public static Iterable getInstances() {
private boolean indexDisabled;
+ private boolean checkSearchForReuseTimeout = false;
+
+ private final Duration searchForReuseTimeout;
+
+ private long searchForReuseStartTimeEpoc;
+
private int depth;
private Collection unshallowObjects;
@@ -356,6 +364,7 @@ public PackWriter(PackConfig config, final ObjectReader reader,
deltaBaseAsOffset = config.isDeltaBaseAsOffset();
reuseDeltas = config.isReuseDeltas();
+ searchForReuseTimeout = config.getSearchForReuseTimeout();
reuseValidate = true; // be paranoid by default
stats = statsAccumulator != null ? statsAccumulator
: new PackStatistics.Accumulator();
@@ -404,6 +413,24 @@ public boolean isDeltaBaseAsOffset() {
return deltaBaseAsOffset;
}
+ /**
+ * Check whether the search for reuse phase is taking too long. This could
+ * be the case when the number of objects and pack files is high and the
+ * system is under pressure. If that's the case and
+ * checkSearchForReuseTimeout is true abort the search.
+ *
+ * @throws SearchForReuseTimeout
+ * if the search for reuse is taking too long.
+ */
+ public void checkSearchForReuseTimeout() throws SearchForReuseTimeout {
+ if (checkSearchForReuseTimeout
+ && Duration.ofMillis(System.currentTimeMillis()
+ - searchForReuseStartTimeEpoc)
+ .compareTo(searchForReuseTimeout) > 0) {
+ throw new SearchForReuseTimeout(searchForReuseTimeout);
+ }
+ }
+
/**
* Set writer delta base format. Delta base can be written as an offset in a
* pack file (new approach reducing file size) or as an object id (legacy
@@ -419,6 +446,22 @@ public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) {
this.deltaBaseAsOffset = deltaBaseAsOffset;
}
+ /**
+ * Set the writer to check for long search for reuse, exceeding the timeout.
+ * Selecting an object representation can be an expensive operation. It is
+ * possible to set a max search for reuse time (see
+ * PackConfig#CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT for more details).
+ *
+ * However some operations, i.e.: GC, need to find the best candidate
+ * regardless how much time the operation will need to finish.
+ *
+ * This method enables the search for reuse timeout check, otherwise
+ * disabled.
+ */
+ public void enableSearchForReuseTimeout() {
+ this.checkSearchForReuseTimeout = true;
+ }
+
/**
* Check if the writer will reuse commits that are already stored as deltas.
*
@@ -1306,6 +1349,7 @@ private void searchForReuse(ProgressMonitor monitor) throws IOException {
cnt += objectsLists[OBJ_TAG].size();
long start = System.currentTimeMillis();
+ searchForReuseStartTimeEpoc = start;
beginPhase(PackingPhase.FINDING_SOURCES, monitor, cnt);
if (cnt <= 4096) {
// For small object counts, do everything as one list.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index 3e3d9b569..b6f4798dc 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -736,4 +736,12 @@ public final class ConfigConstants {
* @since 5.11
*/
public static final String CONFIG_KEY_DEFAULT_BRANCH = "defaultbranch";
+
+ /**
+ * The "pack.searchForReuseTimeout" key
+ *
+ * @since 5.13
+ */
+ public static final String CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT = "searchforreusetimeout";
+
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
index f76dd2721..6aa8be642 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
@@ -29,6 +29,7 @@
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_DELTAS;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_OBJECTS;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SINGLE_PACK;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_THREADS;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK;
@@ -36,7 +37,9 @@
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION;
+import java.time.Duration;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import java.util.zip.Deflater;
import org.eclipse.jgit.internal.storage.file.PackIndexWriter;
@@ -222,6 +225,16 @@ public class PackConfig {
*/
public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90;
+ /**
+ * Default max time to spend during the search for reuse phase. This
+ * optimization is disabled by default: {@value}
+ *
+ * @see #setSearchForReuseTimeout(Duration)
+ * @since 5.13
+ */
+ public static final Duration DEFAULT_SEARCH_FOR_REUSE_TIMEOUT = Duration
+ .ofSeconds(Integer.MAX_VALUE);
+
private int compressionLevel = Deflater.DEFAULT_COMPRESSION;
private boolean reuseDeltas = DEFAULT_REUSE_DELTAS;
@@ -272,6 +285,8 @@ public class PackConfig {
private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS;
+ private Duration searchForReuseTimeout = DEFAULT_SEARCH_FOR_REUSE_TIMEOUT;
+
private boolean cutDeltaChains;
private boolean singlePack;
@@ -342,6 +357,7 @@ public PackConfig(PackConfig cfg) {
this.bitmapInactiveBranchAgeInDays = cfg.bitmapInactiveBranchAgeInDays;
this.cutDeltaChains = cfg.cutDeltaChains;
this.singlePack = cfg.singlePack;
+ this.searchForReuseTimeout = cfg.searchForReuseTimeout;
}
/**
@@ -1103,6 +1119,18 @@ public int getBitmapInactiveBranchAgeInDays() {
return bitmapInactiveBranchAgeInDays;
}
+ /**
+ * Get the max time to spend during the search for reuse phase.
+ *
+ * Default setting: {@value #DEFAULT_SEARCH_FOR_REUSE_TIMEOUT}
+ *
+ * @return the maximum time to spend during the search for reuse phase.
+ * @since 5.13
+ */
+ public Duration getSearchForReuseTimeout() {
+ return searchForReuseTimeout;
+ }
+
/**
* Set the age in days that marks a branch as "inactive".
*
@@ -1116,6 +1144,19 @@ public void setBitmapInactiveBranchAgeInDays(int ageInDays) {
bitmapInactiveBranchAgeInDays = ageInDays;
}
+ /**
+ * Set the max time to spend during the search for reuse phase.
+ *
+ * @param timeout
+ * max time allowed during the search for reuse phase
+ *
+ * Default setting: {@value #DEFAULT_SEARCH_FOR_REUSE_TIMEOUT}
+ * @since 5.13
+ */
+ public void setSearchForReuseTimeout(Duration timeout) {
+ searchForReuseTimeout = timeout;
+ }
+
/**
* Update properties by setting fields from the configuration.
*
@@ -1179,6 +1220,10 @@ public void fromConfig(Config rc) {
setBitmapInactiveBranchAgeInDays(rc.getInt(CONFIG_PACK_SECTION,
CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS,
getBitmapInactiveBranchAgeInDays()));
+ setSearchForReuseTimeout(Duration.ofSeconds(rc.getTimeUnit(
+ CONFIG_PACK_SECTION, null,
+ CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT,
+ getSearchForReuseTimeout().getSeconds(), TimeUnit.SECONDS)));
setWaitPreventRacyPack(rc.getBoolean(CONFIG_PACK_SECTION,
CONFIG_KEY_WAIT_PREVENT_RACYPACK, isWaitPreventRacyPack()));
setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION,
@@ -1216,6 +1261,8 @@ public String toString() {
.append(getBitmapExcessiveBranchCount());
b.append(", bitmapInactiveBranchAge=") //$NON-NLS-1$
.append(getBitmapInactiveBranchAgeInDays());
+ b.append(", searchForReuseTimeout") //$NON-NLS-1$
+ .append(getSearchForReuseTimeout());
b.append(", singlePack=").append(getSinglePack()); //$NON-NLS-1$
return b.toString();
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
index ecf175193..37a1c1e58 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
@@ -2359,6 +2359,7 @@ else if (ref.getName().startsWith(Constants.R_HEADS))
GitProtocolConstants.SECTION_PACKFILE + '\n');
}
}
+ pw.enableSearchForReuseTimeout();
pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut);
if (msgOut != NullOutputStream.INSTANCE) {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java
index 0c74bfdf6..3816d5ed0 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2009, Google Inc. and others
+ * Copyright (C) 2009, 2021 Google Inc. and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -73,6 +73,15 @@ public boolean include(TreeWalk walker) throws MissingObjectException,
}
+ @Override
+ public int matchFilter(TreeWalk walker) throws MissingObjectException,
+ IncorrectObjectTypeException, IOException {
+ if (walker.isSubtree()) {
+ return -1;
+ }
+ return super.matchFilter(walker);
+ }
+
/** {@inheritDoc} */
@Override
public boolean shouldBeRecursive() {
diff --git a/pom.xml b/pom.xml
index 2ba022f39..000be66e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -151,7 +151,7 @@
1.8
${project.build.directory}/META-INF/MANIFEST.MF
- 5.10.0.202012080955-r
+ 5.12.0.202106070339-r
2.6.0
0.1.55
1.1.1
@@ -162,7 +162,7 @@
1.19
4.3.1
3.1.0
- 9.4.40.v20210413
+ 9.4.42.v20210604
0.15.3
4.5.13
4.4.14
@@ -172,9 +172,9 @@
1.7.0
2.8.6
1.65
- 4.2.2
- 3.1.1
- 3.0.0
+ 4.2.3
+ 3.1.2
+ 3.1.1
3.0.0-M5
${maven-surefire-plugin-version}
3.8.1
@@ -247,7 +247,7 @@
org.apache.maven.plugins
maven-dependency-plugin
- 3.1.2
+ 3.2.0
@@ -337,7 +337,7 @@
org.jacoco
jacoco-maven-plugin
- 0.8.6
+ 0.8.7
org.apache.maven.plugins