Teach UploadPack shallow fetch in protocol v2
Add support for the "shallow" and "deepen" parameters in the "fetch" command in the fetch-pack/upload-pack protocol v2. Advertise support for this in the capability advertisement. TODO: implement deepen-relative, deepen-since, deepen-not Change-Id: I7ffd80d6c38872f9d713ac7d6e0412106b3766d7 Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Jonathan Nieder <jrn@google.com>
This commit is contained in:
parent
cd0d69ffec
commit
f7e501c36c
|
@ -377,7 +377,7 @@ private ByteArrayInputStream uploadPackV2(RequestPolicy requestPolicy,
|
|||
// allow additional commands to be added to the list,
|
||||
// and additional capabilities to be added to existing
|
||||
// commands without requiring test changes.
|
||||
hasItems("ls-refs", "fetch"));
|
||||
hasItems("ls-refs", "fetch=shallow"));
|
||||
assertTrue(pckIn.readString() == PacketLineIn.END);
|
||||
return recvStream;
|
||||
}
|
||||
|
@ -877,6 +877,102 @@ public void testV2FetchOfsDelta() throws Exception {
|
|||
assertTrue(stats.getNumOfsDelta() != 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV2FetchShallow() throws Exception {
|
||||
RevCommit commonParent = remote.commit().message("parent").create();
|
||||
RevCommit fooChild = remote.commit().message("x").parent(commonParent).create();
|
||||
RevCommit barChild = remote.commit().message("y").parent(commonParent).create();
|
||||
remote.update("branch1", barChild);
|
||||
|
||||
// Without shallow, the server thinks that we have
|
||||
// commonParent, so it doesn't send it.
|
||||
ByteArrayInputStream recvStream = uploadPackV2(
|
||||
"command=fetch\n",
|
||||
PacketLineIn.DELIM,
|
||||
"want " + barChild.toObjectId().getName() + "\n",
|
||||
"have " + fooChild.toObjectId().getName() + "\n",
|
||||
"done\n",
|
||||
PacketLineIn.END);
|
||||
PacketLineIn pckIn = new PacketLineIn(recvStream);
|
||||
assertThat(pckIn.readString(), is("packfile"));
|
||||
parsePack(recvStream);
|
||||
assertTrue(client.hasObject(barChild.toObjectId()));
|
||||
assertFalse(client.hasObject(commonParent.toObjectId()));
|
||||
|
||||
// With shallow, the server knows that we don't have
|
||||
// commonParent, so it sends it.
|
||||
recvStream = uploadPackV2(
|
||||
"command=fetch\n",
|
||||
PacketLineIn.DELIM,
|
||||
"want " + barChild.toObjectId().getName() + "\n",
|
||||
"have " + fooChild.toObjectId().getName() + "\n",
|
||||
"shallow " + fooChild.toObjectId().getName() + "\n",
|
||||
"done\n",
|
||||
PacketLineIn.END);
|
||||
pckIn = new PacketLineIn(recvStream);
|
||||
assertThat(pckIn.readString(), is("packfile"));
|
||||
parsePack(recvStream);
|
||||
assertTrue(client.hasObject(commonParent.toObjectId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV2FetchDeepenAndDone() throws Exception {
|
||||
RevCommit parent = remote.commit().message("parent").create();
|
||||
RevCommit child = remote.commit().message("x").parent(parent).create();
|
||||
remote.update("branch1", child);
|
||||
|
||||
// "deepen 1" sends only the child.
|
||||
ByteArrayInputStream recvStream = uploadPackV2(
|
||||
"command=fetch\n",
|
||||
PacketLineIn.DELIM,
|
||||
"want " + child.toObjectId().getName() + "\n",
|
||||
"deepen 1\n",
|
||||
"done\n",
|
||||
PacketLineIn.END);
|
||||
PacketLineIn pckIn = new PacketLineIn(recvStream);
|
||||
assertThat(pckIn.readString(), is("shallow-info"));
|
||||
assertThat(pckIn.readString(), is("shallow " + child.toObjectId().getName()));
|
||||
assertThat(pckIn.readString(), theInstance(PacketLineIn.DELIM));
|
||||
assertThat(pckIn.readString(), is("packfile"));
|
||||
parsePack(recvStream);
|
||||
assertTrue(client.hasObject(child.toObjectId()));
|
||||
assertFalse(client.hasObject(parent.toObjectId()));
|
||||
|
||||
// Without that, the parent is sent too.
|
||||
recvStream = uploadPackV2(
|
||||
"command=fetch\n",
|
||||
PacketLineIn.DELIM,
|
||||
"want " + child.toObjectId().getName() + "\n",
|
||||
"done\n",
|
||||
PacketLineIn.END);
|
||||
pckIn = new PacketLineIn(recvStream);
|
||||
assertThat(pckIn.readString(), is("packfile"));
|
||||
parsePack(recvStream);
|
||||
assertTrue(client.hasObject(parent.toObjectId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testV2FetchDeepenWithoutDone() throws Exception {
|
||||
RevCommit parent = remote.commit().message("parent").create();
|
||||
RevCommit child = remote.commit().message("x").parent(parent).create();
|
||||
remote.update("branch1", child);
|
||||
|
||||
ByteArrayInputStream recvStream = uploadPackV2(
|
||||
"command=fetch\n",
|
||||
PacketLineIn.DELIM,
|
||||
"want " + child.toObjectId().getName() + "\n",
|
||||
"deepen 1\n",
|
||||
PacketLineIn.END);
|
||||
PacketLineIn pckIn = new PacketLineIn(recvStream);
|
||||
|
||||
// Verify that only the correct section is sent. "shallow-info"
|
||||
// is not sent because, according to the specification, it is
|
||||
// sent only if a packfile is sent.
|
||||
assertThat(pckIn.readString(), is("acknowledgments"));
|
||||
assertThat(pckIn.readString(), is("NAK"));
|
||||
assertThat(pckIn.readString(), theInstance(PacketLineIn.END));
|
||||
}
|
||||
|
||||
private static class RejectAllRefFilter implements RefFilter {
|
||||
@Override
|
||||
public Map<String, Ref> filter(Map<String, Ref> refs) {
|
||||
|
|
|
@ -224,6 +224,8 @@ credentialPassword=Password
|
|||
credentialUsername=Username
|
||||
daemonAlreadyRunning=Daemon already running
|
||||
daysAgo={0} days ago
|
||||
deepenNotWithDeepen=Cannot combine deepen with deepen-not
|
||||
deepenSinceWithDeepen=Cannot combine deepen with deepen-since
|
||||
deleteBranchUnexpectedResult=Delete branch returned unexpected result {0}
|
||||
deleteFileFailed=Could not delete file {0}
|
||||
deleteRequiresZeroNewId=Delete requires new ID to be zero
|
||||
|
@ -395,6 +397,7 @@ invalidStageForPath=Invalid stage {0} for path {1}
|
|||
invalidSystemProperty=Invalid system property ''{0}'': ''{1}''; using default value {2}
|
||||
invalidTagOption=Invalid tag option: {0}
|
||||
invalidTimeout=Invalid timeout: {0}
|
||||
invalidTimestamp=Invalid timestamp in {0}
|
||||
invalidTimeUnitValue2=Invalid time unit value: {0}.{1}={2}
|
||||
invalidTimeUnitValue3=Invalid time unit value: {0}.{1}.{2}={3}
|
||||
invalidTreeZeroLengthName=Cannot append a tree entry with zero-length name
|
||||
|
|
|
@ -285,6 +285,8 @@ public static JGitText get() {
|
|||
/***/ public String credentialUsername;
|
||||
/***/ public String daemonAlreadyRunning;
|
||||
/***/ public String daysAgo;
|
||||
/***/ public String deepenNotWithDeepen;
|
||||
/***/ public String deepenSinceWithDeepen;
|
||||
/***/ public String deleteBranchUnexpectedResult;
|
||||
/***/ public String deleteFileFailed;
|
||||
/***/ public String deleteRequiresZeroNewId;
|
||||
|
@ -455,6 +457,7 @@ public static JGitText get() {
|
|||
/***/ public String invalidSystemProperty;
|
||||
/***/ public String invalidTagOption;
|
||||
/***/ public String invalidTimeout;
|
||||
/***/ public String invalidTimestamp;
|
||||
/***/ public String invalidTimeUnitValue2;
|
||||
/***/ public String invalidTimeUnitValue3;
|
||||
/***/ public String invalidTreeZeroLengthName;
|
||||
|
|
|
@ -107,6 +107,14 @@ public class GitProtocolConstants {
|
|||
*/
|
||||
public static final String OPTION_SHALLOW = "shallow"; //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* The client wants the "deepen" command to be interpreted as relative to
|
||||
* the client's shallow commits.
|
||||
*
|
||||
* @since 5.0
|
||||
*/
|
||||
public static final String OPTION_DEEPEN_RELATIVE = "deepen-relative"; //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* The client does not want progress messages and will ignore them.
|
||||
*
|
||||
|
|
|
@ -50,6 +50,7 @@
|
|||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
|
||||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
|
||||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
|
||||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_DEEPEN_RELATIVE;
|
||||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_FILTER;
|
||||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG;
|
||||
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK;
|
||||
|
@ -123,7 +124,7 @@ public class UploadPack {
|
|||
private static final String[] v2CapabilityAdvertisement = {
|
||||
"version 2", //$NON-NLS-1$
|
||||
COMMAND_LS_REFS,
|
||||
COMMAND_FETCH
|
||||
COMMAND_FETCH + '=' + OPTION_SHALLOW
|
||||
};
|
||||
|
||||
/** Policy the server uses to validate client requests */
|
||||
|
@ -302,6 +303,19 @@ public Set<String> getOptions() {
|
|||
/** Desired depth from the client on a shallow request. */
|
||||
private int depth;
|
||||
|
||||
/**
|
||||
* Commit time of the newest objects the client has asked us using
|
||||
* --shallow-since not to send. Cannot be nonzero if depth is nonzero.
|
||||
*/
|
||||
private int shallowSince;
|
||||
|
||||
/**
|
||||
* (Possibly short) ref names, ancestors of which the client has asked us
|
||||
* not to send using --shallow-exclude. Cannot be non-null if depth is
|
||||
* nonzero.
|
||||
*/
|
||||
private @Nullable List<String> shallowExcludeRefs;
|
||||
|
||||
/** Commit time of the oldest common commit, in seconds. */
|
||||
private int oldestTime;
|
||||
|
||||
|
@ -813,7 +827,7 @@ else if (requestValidator instanceof AnyRequestValidator)
|
|||
if (!clientShallowCommits.isEmpty())
|
||||
verifyClientShallow();
|
||||
if (depth != 0)
|
||||
processShallow(unshallowCommits);
|
||||
processShallow(null, unshallowCommits, true);
|
||||
if (!clientShallowCommits.isEmpty())
|
||||
walk.assumeShallow(clientShallowCommits);
|
||||
sendPack = negotiate(accumulator);
|
||||
|
@ -968,12 +982,65 @@ private void fetchV2() throws IOException {
|
|||
includeTag = true;
|
||||
} else if (line.equals(OPTION_OFS_DELTA)) {
|
||||
options.add(OPTION_OFS_DELTA);
|
||||
} else if (line.startsWith("shallow ")) { //$NON-NLS-1$
|
||||
clientShallowCommits.add(ObjectId.fromString(line.substring(8)));
|
||||
} else if (line.startsWith("deepen ")) { //$NON-NLS-1$
|
||||
depth = Integer.parseInt(line.substring(7));
|
||||
if (depth <= 0) {
|
||||
throw new PackProtocolException(
|
||||
MessageFormat.format(JGitText.get().invalidDepth,
|
||||
Integer.valueOf(depth)));
|
||||
}
|
||||
if (shallowSince != 0) {
|
||||
throw new PackProtocolException(
|
||||
JGitText.get().deepenSinceWithDeepen);
|
||||
}
|
||||
if (shallowExcludeRefs != null) {
|
||||
throw new PackProtocolException(
|
||||
JGitText.get().deepenNotWithDeepen);
|
||||
}
|
||||
} else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$
|
||||
List<String> exclude = shallowExcludeRefs;
|
||||
if (exclude == null) {
|
||||
exclude = shallowExcludeRefs = new ArrayList<>();
|
||||
}
|
||||
exclude.add(line.substring(11));
|
||||
if (depth != 0) {
|
||||
throw new PackProtocolException(
|
||||
JGitText.get().deepenNotWithDeepen);
|
||||
}
|
||||
} else if (line.equals(OPTION_DEEPEN_RELATIVE)) {
|
||||
options.add(OPTION_DEEPEN_RELATIVE);
|
||||
} else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$
|
||||
shallowSince = Integer.parseInt(line.substring(13));
|
||||
if (shallowSince <= 0) {
|
||||
throw new PackProtocolException(
|
||||
MessageFormat.format(
|
||||
JGitText.get().invalidTimestamp, line));
|
||||
}
|
||||
if (depth != 0) {
|
||||
throw new PackProtocolException(
|
||||
JGitText.get().deepenSinceWithDeepen);
|
||||
}
|
||||
}
|
||||
// else ignore it
|
||||
}
|
||||
rawOut.stopBuffering();
|
||||
|
||||
boolean sectionSent = false;
|
||||
@Nullable List<ObjectId> shallowCommits = null;
|
||||
List<ObjectId> unshallowCommits = new ArrayList<>();
|
||||
|
||||
if (!clientShallowCommits.isEmpty()) {
|
||||
verifyClientShallow();
|
||||
}
|
||||
if (depth != 0 || shallowSince != 0 || shallowExcludeRefs != null) {
|
||||
shallowCommits = new ArrayList<ObjectId>();
|
||||
processShallow(shallowCommits, unshallowCommits, false);
|
||||
}
|
||||
if (!clientShallowCommits.isEmpty())
|
||||
walk.assumeShallow(clientShallowCommits);
|
||||
|
||||
if (doneReceived) {
|
||||
processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE));
|
||||
} else {
|
||||
|
@ -991,7 +1058,21 @@ private void fetchV2() throws IOException {
|
|||
}
|
||||
sectionSent = true;
|
||||
}
|
||||
|
||||
if (doneReceived || okToGiveUp()) {
|
||||
if (shallowCommits != null) {
|
||||
if (sectionSent)
|
||||
pckOut.writeDelim();
|
||||
pckOut.writeString("shallow-info\n"); //$NON-NLS-1$
|
||||
for (ObjectId o : shallowCommits) {
|
||||
pckOut.writeString("shallow " + o.getName() + '\n'); //$NON-NLS-1$
|
||||
}
|
||||
for (ObjectId o : unshallowCommits) {
|
||||
pckOut.writeString("unshallow " + o.getName() + '\n'); //$NON-NLS-1$
|
||||
}
|
||||
sectionSent = true;
|
||||
}
|
||||
|
||||
if (sectionSent)
|
||||
pckOut.writeDelim();
|
||||
pckOut.writeString("packfile\n"); //$NON-NLS-1$
|
||||
|
@ -1078,9 +1159,21 @@ private static Set<ObjectId> refIdSet(Collection<Ref> refs) {
|
|||
|
||||
/*
|
||||
* Determines what "shallow" and "unshallow" lines to send to the user.
|
||||
* The information is written to pckOut and unshallowCommits.
|
||||
* The information is written to shallowCommits (if not null) and
|
||||
* unshallowCommits, and also written to #pckOut (if writeToPckOut is
|
||||
* true).
|
||||
*/
|
||||
private void processShallow(List<ObjectId> unshallowCommits) throws IOException {
|
||||
private void processShallow(@Nullable List<ObjectId> shallowCommits,
|
||||
List<ObjectId> unshallowCommits,
|
||||
boolean writeToPckOut) throws IOException {
|
||||
if (options.contains(OPTION_DEEPEN_RELATIVE) ||
|
||||
shallowSince != 0 ||
|
||||
shallowExcludeRefs != null) {
|
||||
// TODO(jonathantanmy): Implement deepen-relative, deepen-since,
|
||||
// and deepen-not.
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
int walkDepth = depth - 1;
|
||||
try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk(
|
||||
walk.getObjectReader(), walkDepth)) {
|
||||
|
@ -1101,19 +1194,29 @@ private void processShallow(List<ObjectId> unshallowCommits) throws IOException
|
|||
// Commits at the boundary which aren't already shallow in
|
||||
// the client need to be marked as such
|
||||
if (c.getDepth() == walkDepth
|
||||
&& !clientShallowCommits.contains(c))
|
||||
pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
|
||||
&& !clientShallowCommits.contains(c)) {
|
||||
if (shallowCommits != null) {
|
||||
shallowCommits.add(c.copy());
|
||||
}
|
||||
if (writeToPckOut) {
|
||||
pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
|
||||
}
|
||||
}
|
||||
|
||||
// Commits not on the boundary which are shallow in the client
|
||||
// need to become unshallowed
|
||||
if (c.getDepth() < walkDepth
|
||||
&& clientShallowCommits.remove(c)) {
|
||||
unshallowCommits.add(c.copy());
|
||||
pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
|
||||
if (writeToPckOut) {
|
||||
pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pckOut.end();
|
||||
if (writeToPckOut) {
|
||||
pckOut.end();
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyClientShallow()
|
||||
|
|
Loading…
Reference in New Issue