Fix concurrent creation of fan-out object directories
If multiple threads attempted to insert loose objects into the same new fan-out directory, the creation of that directory was subject to a race condition that could lead to an unnecessary IOException being thrown - because an inserter could not 'create' a directory that had just been generated by a different thread. All we require is that the directory does indeed *exist*, so not being able to _create_ it is not actually a fatal problem. Setting 'skipExisting' to 'true' on the call to mkdir() fixes the issue. I found this issue as a real world occurrence while working on The BFG Repo Cleaner (https://github.com/rtyley/bfg-repo-cleaner), a tool which concurrently performs a lot of object creation. In order to demonstrate the problem here I've added a small test case which reliably reproduces the issue on the few different hardware systems I've tried. The error thrown when the race-condition arises is this: java.io.IOException: Creating directory /home/roberto/repo.git/objects/e6 failed at org.eclipse.jgit.util.FileUtils.mkdir(FileUtils.java:182) at org.eclipse.jgit.storage.file.ObjectDirectory.insertUnpackedObject(ObjectDirectory.java:590) at org.eclipse.jgit.storage.file.ObjectDirectoryInserter.insertOneObject(ObjectDirectoryInserter.java:113) at org.eclipse.jgit.storage.file.ObjectDirectoryInserter.insert(ObjectDirectoryInserter.java:91) at org.eclipse.jgit.lib.ObjectInserter.insert(ObjectInserter.java:329) Change-Id: I88eac49bc600c56ba9ad290e6133d8a7113125ab
This commit is contained in:
parent
912ef3da19
commit
5dcc8693d7
|
@ -0,0 +1,77 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2012, Roberto Tyley <roberto.tyley@gmail.com>
|
||||||
|
*
|
||||||
|
* This program and the accompanying materials are made available
|
||||||
|
* under the terms of the Eclipse Distribution License v1.0 which
|
||||||
|
* accompanies this distribution, is reproduced below, and is
|
||||||
|
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||||
|
*
|
||||||
|
* All rights reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or
|
||||||
|
* without modification, are permitted provided that the following
|
||||||
|
* conditions are met:
|
||||||
|
*
|
||||||
|
* - Redistributions of source code must retain the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* - Redistributions in binary form must reproduce the above
|
||||||
|
* copyright notice, this list of conditions and the following
|
||||||
|
* disclaimer in the documentation and/or other materials provided
|
||||||
|
* with the distribution.
|
||||||
|
*
|
||||||
|
* - Neither the name of the Eclipse Foundation, Inc. nor the
|
||||||
|
* names of its contributors may be used to endorse or promote
|
||||||
|
* products derived from this software without specific prior
|
||||||
|
* written permission.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
||||||
|
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||||
|
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||||
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||||
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||||
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||||
|
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||||
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.eclipse.jgit.storage.file;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
|
import org.eclipse.jgit.lib.RepositoryTestCase;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.*;
|
||||||
|
|
||||||
|
public class ObjectDirectoryTest extends RepositoryTestCase {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConcurrentInsertionOfBlobsToTheSameNewFanOutDirectory()
|
||||||
|
throws Exception {
|
||||||
|
ExecutorService e = Executors.newCachedThreadPool();
|
||||||
|
for (int i=0; i < 100; ++i) {
|
||||||
|
ObjectDirectory db = createBareRepository().getObjectDatabase();
|
||||||
|
for (Future f : e.invokeAll(blobInsertersForTheSameFanOutDir(db))) {
|
||||||
|
f.get();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Collection<Callable<ObjectId>> blobInsertersForTheSameFanOutDir(
|
||||||
|
final ObjectDirectory db) {
|
||||||
|
Callable<ObjectId> callable = new Callable<ObjectId>() {
|
||||||
|
public ObjectId call() throws Exception {
|
||||||
|
return db.newInserter().insert(Constants.OBJ_BLOB, new byte[0]);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return Collections.nCopies(4, callable);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -577,7 +577,7 @@ InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id,
|
||||||
// directories are always lazily created. Note that we
|
// directories are always lazily created. Note that we
|
||||||
// try the rename first as the directory likely does exist.
|
// try the rename first as the directory likely does exist.
|
||||||
//
|
//
|
||||||
FileUtils.mkdir(dst.getParentFile());
|
FileUtils.mkdir(dst.getParentFile(), true);
|
||||||
if (tmp.renameTo(dst)) {
|
if (tmp.renameTo(dst)) {
|
||||||
dst.setReadOnly();
|
dst.setReadOnly();
|
||||||
unpackedObjectCache.add(id);
|
unpackedObjectCache.add(id);
|
||||||
|
|
Loading…
Reference in New Issue