Speed up clone/fetch with large number of refs
Instead of re-reading all refs after each update, execute the deletes first, then read all refs once and perform the check for conflicting ref names in memory. Change-Id: I17d0b3ccc27f868c8497607d8e57bf7082e65ba3
This commit is contained in:
parent
4796fe7043
commit
5cf53fdacf
|
@ -60,6 +60,8 @@
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
@ -67,17 +69,20 @@
|
|||
import org.eclipse.jgit.events.ListenerHandle;
|
||||
import org.eclipse.jgit.events.RefsChangedEvent;
|
||||
import org.eclipse.jgit.events.RefsChangedListener;
|
||||
import org.eclipse.jgit.internal.storage.file.FileRepository;
|
||||
import org.eclipse.jgit.internal.storage.file.RefDirectory;
|
||||
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
|
||||
import org.eclipse.jgit.junit.TestRepository;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.BatchRefUpdate;
|
||||
import org.eclipse.jgit.lib.NullProgressMonitor;
|
||||
import org.eclipse.jgit.lib.Ref;
|
||||
import org.eclipse.jgit.lib.Ref.Storage;
|
||||
import org.eclipse.jgit.lib.RefDatabase;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.revwalk.RevCommit;
|
||||
import org.eclipse.jgit.revwalk.RevTag;
|
||||
import org.eclipse.jgit.revwalk.RevWalk;
|
||||
import org.eclipse.jgit.transport.ReceiveCommand;
|
||||
import org.eclipse.jgit.transport.ReceiveCommand.Type;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
|
@ -1175,6 +1180,112 @@ public void onRefsChanged(RefsChangedEvent event) {
|
|||
assertEquals(1, changeCount.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchRefUpdateSimpleNoForce() throws IOException {
|
||||
writeLooseRef("refs/heads/master", A);
|
||||
writeLooseRef("refs/heads/masters", B);
|
||||
List<ReceiveCommand> commands = Arrays.asList(
|
||||
newCommand(A, B, "refs/heads/master",
|
||||
ReceiveCommand.Type.UPDATE),
|
||||
newCommand(B, A, "refs/heads/masters",
|
||||
ReceiveCommand.Type.UPDATE_NONFASTFORWARD));
|
||||
BatchRefUpdate batchUpdate = refdir.newBatchUpdate();
|
||||
batchUpdate.addCommand(commands);
|
||||
batchUpdate
|
||||
.execute(new RevWalk(diskRepo), NullProgressMonitor.INSTANCE);
|
||||
Map<String, Ref> refs = refdir.getRefs(RefDatabase.ALL);
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(0).getResult());
|
||||
assertEquals(ReceiveCommand.Result.REJECTED_NONFASTFORWARD, commands
|
||||
.get(1).getResult());
|
||||
assertEquals("[HEAD, refs/heads/master, refs/heads/masters]", refs
|
||||
.keySet().toString());
|
||||
assertEquals(B.getId(), refs.get("refs/heads/master").getObjectId());
|
||||
assertEquals(B.getId(), refs.get("refs/heads/masters").getObjectId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchRefUpdateSimpleForce() throws IOException {
|
||||
writeLooseRef("refs/heads/master", A);
|
||||
writeLooseRef("refs/heads/masters", B);
|
||||
List<ReceiveCommand> commands = Arrays.asList(
|
||||
newCommand(A, B, "refs/heads/master",
|
||||
ReceiveCommand.Type.UPDATE),
|
||||
newCommand(B, A, "refs/heads/masters",
|
||||
ReceiveCommand.Type.UPDATE_NONFASTFORWARD));
|
||||
BatchRefUpdate batchUpdate = refdir.newBatchUpdate();
|
||||
batchUpdate.setAllowNonFastForwards(true);
|
||||
batchUpdate.addCommand(commands);
|
||||
batchUpdate
|
||||
.execute(new RevWalk(diskRepo), NullProgressMonitor.INSTANCE);
|
||||
Map<String, Ref> refs = refdir.getRefs(RefDatabase.ALL);
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(0).getResult());
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(1).getResult());
|
||||
assertEquals("[HEAD, refs/heads/master, refs/heads/masters]", refs
|
||||
.keySet().toString());
|
||||
assertEquals(B.getId(), refs.get("refs/heads/master").getObjectId());
|
||||
assertEquals(A.getId(), refs.get("refs/heads/masters").getObjectId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchRefUpdateConflict() throws IOException {
|
||||
writeLooseRef("refs/heads/master", A);
|
||||
writeLooseRef("refs/heads/masters", B);
|
||||
List<ReceiveCommand> commands = Arrays.asList(
|
||||
newCommand(A, B, "refs/heads/master",
|
||||
ReceiveCommand.Type.UPDATE),
|
||||
newCommand(null, A, "refs/heads/master/x",
|
||||
ReceiveCommand.Type.CREATE),
|
||||
newCommand(null, A, "refs/heads", ReceiveCommand.Type.CREATE));
|
||||
BatchRefUpdate batchUpdate = refdir.newBatchUpdate();
|
||||
batchUpdate.setAllowNonFastForwards(true);
|
||||
batchUpdate.addCommand(commands);
|
||||
batchUpdate
|
||||
.execute(new RevWalk(diskRepo), NullProgressMonitor.INSTANCE);
|
||||
Map<String, Ref> refs = refdir.getRefs(RefDatabase.ALL);
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(0).getResult());
|
||||
assertEquals(ReceiveCommand.Result.LOCK_FAILURE, commands.get(1)
|
||||
.getResult());
|
||||
assertEquals(ReceiveCommand.Result.LOCK_FAILURE, commands.get(2)
|
||||
.getResult());
|
||||
assertEquals("[HEAD, refs/heads/master, refs/heads/masters]", refs
|
||||
.keySet().toString());
|
||||
assertEquals(B.getId(), refs.get("refs/heads/master").getObjectId());
|
||||
assertEquals(B.getId(), refs.get("refs/heads/masters").getObjectId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBatchRefUpdateConflictThanksToDelete() throws IOException {
|
||||
writeLooseRef("refs/heads/master", A);
|
||||
writeLooseRef("refs/heads/masters", B);
|
||||
List<ReceiveCommand> commands = Arrays.asList(
|
||||
newCommand(A, B, "refs/heads/master",
|
||||
ReceiveCommand.Type.UPDATE),
|
||||
newCommand(null, A, "refs/heads/masters/x",
|
||||
ReceiveCommand.Type.CREATE),
|
||||
newCommand(B, null, "refs/heads/masters",
|
||||
ReceiveCommand.Type.DELETE));
|
||||
BatchRefUpdate batchUpdate = refdir.newBatchUpdate();
|
||||
batchUpdate.setAllowNonFastForwards(true);
|
||||
batchUpdate.addCommand(commands);
|
||||
batchUpdate
|
||||
.execute(new RevWalk(diskRepo), NullProgressMonitor.INSTANCE);
|
||||
Map<String, Ref> refs = refdir.getRefs(RefDatabase.ALL);
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(0).getResult());
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(1).getResult());
|
||||
assertEquals(ReceiveCommand.Result.OK, commands.get(2).getResult());
|
||||
assertEquals("[HEAD, refs/heads/master, refs/heads/masters/x]", refs
|
||||
.keySet().toString());
|
||||
assertEquals(A.getId(), refs.get("refs/heads/masters/x").getObjectId());
|
||||
}
|
||||
|
||||
private static ReceiveCommand newCommand(RevCommit a, RevCommit b,
|
||||
String string, Type update) {
|
||||
ReceiveCommand ret = new ReceiveCommand(a != null ? a.getId() : null,
|
||||
b != null ? b.getId() : null, string, update);
|
||||
ret.setResult(ReceiveCommand.Result.NOT_ATTEMPTED);
|
||||
return ret;
|
||||
}
|
||||
|
||||
private void writeLooseRef(String name, AnyObjectId id) throws IOException {
|
||||
writeLooseRef(name, id.name() + "\n");
|
||||
}
|
||||
|
|
|
@ -53,9 +53,11 @@
|
|||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
import org.eclipse.jgit.internal.JGitText;
|
||||
import org.eclipse.jgit.lib.RefUpdate.Result;
|
||||
import org.eclipse.jgit.revwalk.RevWalk;
|
||||
import org.eclipse.jgit.transport.ReceiveCommand;
|
||||
|
||||
|
@ -242,41 +244,120 @@ public BatchRefUpdate addCommand(Collection<ReceiveCommand> cmd) {
|
|||
* @param walk
|
||||
* a RevWalk to parse tags in case the storage system wants to
|
||||
* store them pre-peeled, a common performance optimization.
|
||||
* @param update
|
||||
* @param monitor
|
||||
* progress monitor to receive update status on.
|
||||
* @throws IOException
|
||||
* the database is unable to accept the update. Individual
|
||||
* command status must be tested to determine if there is a
|
||||
* partial failure, or a total failure.
|
||||
*/
|
||||
public void execute(RevWalk walk, ProgressMonitor update)
|
||||
public void execute(RevWalk walk, ProgressMonitor monitor)
|
||||
throws IOException {
|
||||
update.beginTask(JGitText.get().updatingReferences, commands.size());
|
||||
monitor.beginTask(JGitText.get().updatingReferences, commands.size());
|
||||
List<ReceiveCommand> commands2 = new ArrayList<ReceiveCommand>(
|
||||
commands.size());
|
||||
List<String> namesToCheck = new ArrayList<String>(commands.size());
|
||||
// First delete refs. This may free the name space for some of the
|
||||
// updates.
|
||||
for (ReceiveCommand cmd : commands) {
|
||||
try {
|
||||
update.update(1);
|
||||
|
||||
if (cmd.getResult() == NOT_ATTEMPTED) {
|
||||
cmd.updateType(walk);
|
||||
RefUpdate ru = newUpdate(cmd);
|
||||
switch (cmd.getType()) {
|
||||
case DELETE:
|
||||
cmd.setResult(ru.delete(walk));
|
||||
continue;
|
||||
|
||||
case CREATE:
|
||||
namesToCheck.add(cmd.getRefName());
|
||||
commands2.add(cmd);
|
||||
break;
|
||||
case UPDATE:
|
||||
case UPDATE_NONFASTFORWARD:
|
||||
cmd.setResult(ru.update(walk));
|
||||
continue;
|
||||
commands2.add(cmd);
|
||||
break;
|
||||
case DELETE:
|
||||
RefUpdate rud = newUpdate(cmd);
|
||||
monitor.update(1);
|
||||
cmd.setResult(rud.delete(walk));
|
||||
}
|
||||
}
|
||||
} catch (IOException err) {
|
||||
cmd.setResult(REJECTED_OTHER_REASON, MessageFormat.format(
|
||||
JGitText.get().lockError, err.getMessage()));
|
||||
cmd.setResult(
|
||||
REJECTED_OTHER_REASON,
|
||||
MessageFormat.format(JGitText.get().lockError,
|
||||
err.getMessage()));
|
||||
}
|
||||
}
|
||||
update.endTask();
|
||||
if (!commands2.isEmpty()) {
|
||||
// What part of the name space is already taken
|
||||
Collection<String> takenNames = new HashSet<String>(refdb.getRefs(
|
||||
RefDatabase.ALL).keySet());
|
||||
Collection<String> takenPrefixes = getTakenPrefixes(takenNames);
|
||||
|
||||
// Now to the update that may require more room in the name space
|
||||
for (ReceiveCommand cmd : commands2) {
|
||||
try {
|
||||
monitor.update(1);
|
||||
|
||||
if (cmd.getResult() == NOT_ATTEMPTED) {
|
||||
cmd.updateType(walk);
|
||||
RefUpdate ru = newUpdate(cmd);
|
||||
SWITCH: switch (cmd.getType()) {
|
||||
case DELETE:
|
||||
// Performed in the first phase
|
||||
break;
|
||||
case UPDATE:
|
||||
case UPDATE_NONFASTFORWARD:
|
||||
monitor.update(1);
|
||||
RefUpdate ruu = newUpdate(cmd);
|
||||
cmd.setResult(ruu.update(walk));
|
||||
break;
|
||||
case CREATE:
|
||||
for (String prefix : getPrefixes(cmd.getRefName())) {
|
||||
if (takenNames.contains(prefix)) {
|
||||
cmd.setResult(Result.LOCK_FAILURE);
|
||||
break SWITCH;
|
||||
}
|
||||
}
|
||||
if (takenPrefixes.contains(cmd.getRefName())) {
|
||||
cmd.setResult(Result.LOCK_FAILURE);
|
||||
break SWITCH;
|
||||
}
|
||||
ru.setCheckConflicting(false);
|
||||
addRefToPrefixes(takenPrefixes, cmd.getRefName());
|
||||
takenNames.add(cmd.getRefName());
|
||||
cmd.setResult(ru.update(walk));
|
||||
}
|
||||
}
|
||||
} catch (IOException err) {
|
||||
cmd.setResult(REJECTED_OTHER_REASON, MessageFormat.format(
|
||||
JGitText.get().lockError, err.getMessage()));
|
||||
}
|
||||
}
|
||||
}
|
||||
monitor.endTask();
|
||||
}
|
||||
|
||||
private static Collection<String> getTakenPrefixes(
|
||||
final Collection<String> names) {
|
||||
Collection<String> ref = new HashSet<String>();
|
||||
for (String name : names)
|
||||
ref.addAll(getPrefixes(name));
|
||||
return ref;
|
||||
}
|
||||
|
||||
private static void addRefToPrefixes(Collection<String> prefixes,
|
||||
String name) {
|
||||
for (String prefix : getPrefixes(name)) {
|
||||
prefixes.add(prefix);
|
||||
}
|
||||
}
|
||||
|
||||
static Collection<String> getPrefixes(String s) {
|
||||
Collection<String> ret = new HashSet<String>();
|
||||
int p1 = s.indexOf('/');
|
||||
while (p1 > 0) {
|
||||
ret.add(s.substring(0, p1));
|
||||
p1 = s.indexOf('/', p1 + 1);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -178,6 +178,8 @@ public static enum Result {
|
|||
*/
|
||||
private boolean detachingSymbolicRef;
|
||||
|
||||
private boolean checkConflicting = true;
|
||||
|
||||
/**
|
||||
* Construct a new update operation for the reference.
|
||||
* <p>
|
||||
|
@ -564,7 +566,7 @@ Result execute(Result status) throws IOException {
|
|||
public Result link(String target) throws IOException {
|
||||
if (!target.startsWith(Constants.R_REFS))
|
||||
throw new IllegalArgumentException(MessageFormat.format(JGitText.get().illegalArgumentNotA, Constants.R_REFS));
|
||||
if (getRefDatabase().isNameConflicting(getName()))
|
||||
if (checkConflicting && getRefDatabase().isNameConflicting(getName()))
|
||||
return Result.LOCK_FAILURE;
|
||||
try {
|
||||
if (!tryLock(false))
|
||||
|
@ -599,7 +601,7 @@ private Result updateImpl(final RevWalk walk, final Store store)
|
|||
RevObject oldObj;
|
||||
|
||||
// don't make expensive conflict check if this is an existing Ref
|
||||
if (oldValue == null && getRefDatabase().isNameConflicting(getName()))
|
||||
if (oldValue == null && checkConflicting && getRefDatabase().isNameConflicting(getName()))
|
||||
return Result.LOCK_FAILURE;
|
||||
try {
|
||||
if (!tryLock(true))
|
||||
|
@ -631,6 +633,16 @@ private Result updateImpl(final RevWalk walk, final Store store)
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable/disable the check for conflicting ref names. By default conflicts
|
||||
* are checked explicitly.
|
||||
*
|
||||
* @param check
|
||||
*/
|
||||
public void setCheckConflicting(boolean check) {
|
||||
checkConflicting = check;
|
||||
}
|
||||
|
||||
private static RevObject safeParse(final RevWalk rw, final AnyObjectId id)
|
||||
throws IOException {
|
||||
try {
|
||||
|
|
Loading…
Reference in New Issue