Retry loose object read upon "Stale file handle" exception
When reading loose objects over NFS it is possible that the OS syscall would fail with ESTALE errors: This happens when the open file descriptor no longer refers to a valid file. Notoriously it is possible to hit this scenario when git data is shared among multiple clients, for example by multiple gerrit instances in HA. If one of the two clients performs a GC operation that would cause the packing and then the pruning of loose objects, the other client might still hold a reference to those objects, which would cause an exception to bubble up the stack. The Linux NFS FAQ[1] (at point A.10), suggests that the proper way to handle such ESTALE scenarios is to: "[...] close the file or directory where the error occurred, and reopen it so the NFS client can resolve the pathname again and retrieve the new file handle." In case of a stale file handle exception, we now attempt to read the loose object again (up to 5 times), until we either succeed or encounter a FileNotFoundException, in which case the search can continue to Packfiles and alternates. The limit of 5 provides an arbitrary upper bounds that is consistent to the one chosen when handling stale file handles for packed-refs files (see [2] for context). [1] http://nfs.sourceforge.net/ [2] https://git.eclipse.org/r/c/jgit/jgit/+/54350 Bug: 573791 Change-Id: I9950002f772bbd8afeb9c6108391923be9d0ef51
This commit is contained in:
parent
12f39c26b0
commit
24d6d60538
|
@ -44,8 +44,12 @@
|
||||||
|
|
||||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
|
import static org.junit.Assert.assertNull;
|
||||||
import static org.junit.Assert.assertThrows;
|
import static org.junit.Assert.assertThrows;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -69,6 +73,7 @@
|
||||||
import org.eclipse.jgit.util.FS;
|
import org.eclipse.jgit.util.FS;
|
||||||
import org.junit.Assume;
|
import org.junit.Assume;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
public class ObjectDirectoryTest extends RepositoryTestCase {
|
public class ObjectDirectoryTest extends RepositoryTestCase {
|
||||||
|
|
||||||
|
@ -194,6 +199,42 @@ public void testShallowFile()
|
||||||
assertTrue(shallowCommits.isEmpty());
|
assertTrue(shallowCommits.isEmpty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOpenLooseObjectSuppressStaleFileHandleException()
|
||||||
|
throws Exception {
|
||||||
|
ObjectId id = ObjectId
|
||||||
|
.fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b");
|
||||||
|
WindowCursor curs = new WindowCursor(db.getObjectDatabase());
|
||||||
|
|
||||||
|
ObjectDirectory mock = mock(ObjectDirectory.class);
|
||||||
|
UnpackedObjectCache unpackedObjectCacheMock = mock(
|
||||||
|
UnpackedObjectCache.class);
|
||||||
|
|
||||||
|
Mockito.when(mock.getObjectLoader(any(), any(), any()))
|
||||||
|
.thenThrow(new IOException("Stale File Handle"));
|
||||||
|
Mockito.when(mock.openLooseObject(curs, id)).thenCallRealMethod();
|
||||||
|
Mockito.when(mock.unpackedObjectCache())
|
||||||
|
.thenReturn(unpackedObjectCacheMock);
|
||||||
|
|
||||||
|
assertNull(mock.openLooseObject(curs, id));
|
||||||
|
verify(unpackedObjectCacheMock).remove(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOpenLooseObjectPropagatesIOExceptions() throws Exception {
|
||||||
|
ObjectId id = ObjectId
|
||||||
|
.fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b");
|
||||||
|
WindowCursor curs = new WindowCursor(db.getObjectDatabase());
|
||||||
|
|
||||||
|
ObjectDirectory mock = mock(ObjectDirectory.class);
|
||||||
|
|
||||||
|
Mockito.when(mock.getObjectLoader(any(), any(), any()))
|
||||||
|
.thenThrow(new IOException("some IO failure"));
|
||||||
|
Mockito.when(mock.openLooseObject(curs, id)).thenCallRealMethod();
|
||||||
|
|
||||||
|
assertThrows(IOException.class, () -> mock.openLooseObject(curs, id));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testShallowFileCorrupt() throws Exception {
|
public void testShallowFileCorrupt() throws Exception {
|
||||||
FileRepository repository = createBareRepository();
|
FileRepository repository = createBareRepository();
|
||||||
|
|
|
@ -416,6 +416,7 @@ logInconsistentFiletimeDiff={}: inconsistent duration from file timestamps on {}
|
||||||
logLargerFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: diff = {} > {} (last good value). Aborting measurement.
|
logLargerFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: diff = {} > {} (last good value). Aborting measurement.
|
||||||
logSmallerFiletime={}: got smaller file timestamp on {}, {}: {} < {}. Aborting measurement at resolution {}.
|
logSmallerFiletime={}: got smaller file timestamp on {}, {}: {} < {}. Aborting measurement at resolution {}.
|
||||||
logXDGConfigHomeInvalid=Environment variable XDG_CONFIG_HOME contains an invalid path {}
|
logXDGConfigHomeInvalid=Environment variable XDG_CONFIG_HOME contains an invalid path {}
|
||||||
|
looseObjectHandleIsStale=loose-object {0} file handle is stale. retry {1} of {2}
|
||||||
maxCountMustBeNonNegative=max count must be >= 0
|
maxCountMustBeNonNegative=max count must be >= 0
|
||||||
mergeConflictOnNonNoteEntries=Merge conflict on non-note entries: base = {0}, ours = {1}, theirs = {2}
|
mergeConflictOnNonNoteEntries=Merge conflict on non-note entries: base = {0}, ours = {1}, theirs = {2}
|
||||||
mergeConflictOnNotes=Merge conflict on note {0}. base = {1}, ours = {2}, theirs = {2}
|
mergeConflictOnNotes=Merge conflict on note {0}. base = {1}, ours = {2}, theirs = {2}
|
||||||
|
|
|
@ -444,6 +444,7 @@ public static JGitText get() {
|
||||||
/***/ public String logLargerFiletimeDiff;
|
/***/ public String logLargerFiletimeDiff;
|
||||||
/***/ public String logSmallerFiletime;
|
/***/ public String logSmallerFiletime;
|
||||||
/***/ public String logXDGConfigHomeInvalid;
|
/***/ public String logXDGConfigHomeInvalid;
|
||||||
|
/***/ public String looseObjectHandleIsStale;
|
||||||
/***/ public String maxCountMustBeNonNegative;
|
/***/ public String maxCountMustBeNonNegative;
|
||||||
/***/ public String mergeConflictOnNonNoteEntries;
|
/***/ public String mergeConflictOnNonNoteEntries;
|
||||||
/***/ public String mergeConflictOnNotes;
|
/***/ public String mergeConflictOnNotes;
|
||||||
|
|
|
@ -85,6 +85,10 @@ public class ObjectDirectory extends FileObjectDatabase {
|
||||||
/** Maximum number of candidates offered as resolutions of abbreviation. */
|
/** Maximum number of candidates offered as resolutions of abbreviation. */
|
||||||
private static final int RESOLVE_ABBREV_LIMIT = 256;
|
private static final int RESOLVE_ABBREV_LIMIT = 256;
|
||||||
|
|
||||||
|
/** Maximum number of attempts to read a loose object for which a stale file
|
||||||
|
* handle exception is thrown */
|
||||||
|
final static int MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS = 5;
|
||||||
|
|
||||||
private final AlternateHandle handle = new AlternateHandle(this);
|
private final AlternateHandle handle = new AlternateHandle(this);
|
||||||
|
|
||||||
private final Config config;
|
private final Config config;
|
||||||
|
@ -212,7 +216,7 @@ public PackInserter newPackInserter() {
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
@Override
|
@Override
|
||||||
public void close() {
|
public void close() {
|
||||||
unpackedObjectCache.clear();
|
unpackedObjectCache().clear();
|
||||||
|
|
||||||
final PackList packs = packList.get();
|
final PackList packs = packList.get();
|
||||||
if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) {
|
if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) {
|
||||||
|
@ -277,7 +281,7 @@ public String toString() {
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
@Override
|
@Override
|
||||||
public boolean has(AnyObjectId objectId) {
|
public boolean has(AnyObjectId objectId) {
|
||||||
return unpackedObjectCache.isUnpacked(objectId)
|
return unpackedObjectCache().isUnpacked(objectId)
|
||||||
|| hasPackedInSelfOrAlternate(objectId, null)
|
|| hasPackedInSelfOrAlternate(objectId, null)
|
||||||
|| hasLooseInSelfOrAlternate(objectId, null);
|
|| hasLooseInSelfOrAlternate(objectId, null);
|
||||||
}
|
}
|
||||||
|
@ -395,7 +399,7 @@ private void resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
|
||||||
@Override
|
@Override
|
||||||
ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId)
|
ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (unpackedObjectCache.isUnpacked(objectId)) {
|
if (unpackedObjectCache().isUnpacked(objectId)) {
|
||||||
ObjectLoader ldr = openLooseObject(curs, objectId);
|
ObjectLoader ldr = openLooseObject(curs, objectId);
|
||||||
if (ldr != null) {
|
if (ldr != null) {
|
||||||
return ldr;
|
return ldr;
|
||||||
|
@ -473,23 +477,71 @@ ObjectLoader openPackedObject(WindowCursor curs, AnyObjectId objectId) {
|
||||||
@Override
|
@Override
|
||||||
ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id)
|
ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
File path = fileFor(id);
|
int readAttempts = 0;
|
||||||
try (FileInputStream in = new FileInputStream(path)) {
|
while (readAttempts < MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS) {
|
||||||
unpackedObjectCache.add(id);
|
readAttempts++;
|
||||||
return UnpackedObject.open(in, path, id, curs);
|
File path = fileFor(id);
|
||||||
} catch (FileNotFoundException noFile) {
|
try {
|
||||||
if (path.exists()) {
|
return getObjectLoader(curs, path, id);
|
||||||
throw noFile;
|
} catch (FileNotFoundException noFile) {
|
||||||
|
if (path.exists()) {
|
||||||
|
throw noFile;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
} catch (IOException e) {
|
||||||
|
if (!FileUtils.isStaleFileHandleInCausalChain(e)) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(MessageFormat.format(
|
||||||
|
JGitText.get().looseObjectHandleIsStale, id.name(),
|
||||||
|
Integer.valueOf(readAttempts), Integer.valueOf(
|
||||||
|
MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
unpackedObjectCache.remove(id);
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
unpackedObjectCache().remove(id);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides a loader for an objectId
|
||||||
|
*
|
||||||
|
* @param curs
|
||||||
|
* cursor on the database
|
||||||
|
* @param path
|
||||||
|
* the path of the loose object
|
||||||
|
* @param id
|
||||||
|
* the object id
|
||||||
|
* @return a loader for the loose file object
|
||||||
|
* @throws IOException
|
||||||
|
* when file does not exist or it could not be opened
|
||||||
|
*/
|
||||||
|
ObjectLoader getObjectLoader(WindowCursor curs, File path, AnyObjectId id)
|
||||||
|
throws IOException {
|
||||||
|
try (FileInputStream in = new FileInputStream(path)) {
|
||||||
|
unpackedObjectCache().add(id);
|
||||||
|
return UnpackedObject.open(in, path, id, curs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <p>
|
||||||
|
* Getter for the field <code>unpackedObjectCache</code>.
|
||||||
|
* </p>
|
||||||
|
* This accessor is particularly useful to allow mocking of this class for
|
||||||
|
* testing purposes.
|
||||||
|
*
|
||||||
|
* @return the cache of the objects currently unpacked.
|
||||||
|
*/
|
||||||
|
UnpackedObjectCache unpackedObjectCache() {
|
||||||
|
return unpackedObjectCache;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
long getObjectSize(WindowCursor curs, AnyObjectId id)
|
long getObjectSize(WindowCursor curs, AnyObjectId id)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (unpackedObjectCache.isUnpacked(id)) {
|
if (unpackedObjectCache().isUnpacked(id)) {
|
||||||
long len = getLooseObjectSize(curs, id);
|
long len = getLooseObjectSize(curs, id);
|
||||||
if (0 <= len) {
|
if (0 <= len) {
|
||||||
return len;
|
return len;
|
||||||
|
@ -567,13 +619,13 @@ private long getLooseObjectSize(WindowCursor curs, AnyObjectId id)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
File f = fileFor(id);
|
File f = fileFor(id);
|
||||||
try (FileInputStream in = new FileInputStream(f)) {
|
try (FileInputStream in = new FileInputStream(f)) {
|
||||||
unpackedObjectCache.add(id);
|
unpackedObjectCache().add(id);
|
||||||
return UnpackedObject.getSize(in, id, curs);
|
return UnpackedObject.getSize(in, id, curs);
|
||||||
} catch (FileNotFoundException noFile) {
|
} catch (FileNotFoundException noFile) {
|
||||||
if (f.exists()) {
|
if (f.exists()) {
|
||||||
throw noFile;
|
throw noFile;
|
||||||
}
|
}
|
||||||
unpackedObjectCache.remove(id);
|
unpackedObjectCache().remove(id);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -667,7 +719,7 @@ InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id,
|
||||||
boolean createDuplicate) throws IOException {
|
boolean createDuplicate) throws IOException {
|
||||||
// If the object is already in the repository, remove temporary file.
|
// If the object is already in the repository, remove temporary file.
|
||||||
//
|
//
|
||||||
if (unpackedObjectCache.isUnpacked(id)) {
|
if (unpackedObjectCache().isUnpacked(id)) {
|
||||||
FileUtils.delete(tmp, FileUtils.RETRY);
|
FileUtils.delete(tmp, FileUtils.RETRY);
|
||||||
return InsertLooseObjectResult.EXISTS_LOOSE;
|
return InsertLooseObjectResult.EXISTS_LOOSE;
|
||||||
}
|
}
|
||||||
|
@ -723,7 +775,7 @@ private InsertLooseObjectResult tryMove(File tmp, File dst,
|
||||||
Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst),
|
Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst),
|
||||||
StandardCopyOption.ATOMIC_MOVE);
|
StandardCopyOption.ATOMIC_MOVE);
|
||||||
dst.setReadOnly();
|
dst.setReadOnly();
|
||||||
unpackedObjectCache.add(id);
|
unpackedObjectCache().add(id);
|
||||||
return InsertLooseObjectResult.INSERTED;
|
return InsertLooseObjectResult.INSERTED;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue