Teach UploadPack shallow fetch in protocol v2

Add support for the "shallow" and "deepen" parameters in the "fetch"
command in the fetch-pack/upload-pack protocol v2. Advertise support for
this in the capability advertisement.

TODO: implement deepen-relative, deepen-since, deepen-not

Change-Id: I7ffd80d6c38872f9d713ac7d6e0412106b3766d7
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Jonathan Nieder <jrn@google.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java

@@ -377,7 +377,7 @@ private ByteArrayInputStream uploadPackV2(RequestPolicy requestPolicy,
 			// allow additional commands to be added to the list,
 			// and additional capabilities to be added to existing
 			// commands without requiring test changes.
-			hasItems("ls-refs", "fetch"));
+			hasItems("ls-refs", "fetch=shallow"));
 		assertTrue(pckIn.readString() == PacketLineIn.END);
 		return recvStream;
 	}
@@ -877,6 +877,102 @@ public void testV2FetchOfsDelta() throws Exception {
 		assertTrue(stats.getNumOfsDelta() != 0);
 	}
 
+	@Test
+	public void testV2FetchShallow() throws Exception {
+		RevCommit commonParent = remote.commit().message("parent").create();
+		RevCommit fooChild = remote.commit().message("x").parent(commonParent).create();
+		RevCommit barChild = remote.commit().message("y").parent(commonParent).create();
+		remote.update("branch1", barChild);
+
+		// Without shallow, the server thinks that we have
+		// commonParent, so it doesn't send it.
+		ByteArrayInputStream recvStream = uploadPackV2(
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"want " + barChild.toObjectId().getName() + "\n",
+			"have " + fooChild.toObjectId().getName() + "\n",
+			"done\n",
+			PacketLineIn.END);
+		PacketLineIn pckIn = new PacketLineIn(recvStream);
+		assertThat(pckIn.readString(), is("packfile"));
+		parsePack(recvStream);
+		assertTrue(client.hasObject(barChild.toObjectId()));
+		assertFalse(client.hasObject(commonParent.toObjectId()));
+
+		// With shallow, the server knows that we don't have
+		// commonParent, so it sends it.
+		recvStream = uploadPackV2(
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"want " + barChild.toObjectId().getName() + "\n",
+			"have " + fooChild.toObjectId().getName() + "\n",
+			"shallow " + fooChild.toObjectId().getName() + "\n",
+			"done\n",
+			PacketLineIn.END);
+		pckIn = new PacketLineIn(recvStream);
+		assertThat(pckIn.readString(), is("packfile"));
+		parsePack(recvStream);
+		assertTrue(client.hasObject(commonParent.toObjectId()));
+	}
+
+	@Test
+	public void testV2FetchDeepenAndDone() throws Exception {
+		RevCommit parent = remote.commit().message("parent").create();
+		RevCommit child = remote.commit().message("x").parent(parent).create();
+		remote.update("branch1", child);
+
+		// "deepen 1" sends only the child.
+		ByteArrayInputStream recvStream = uploadPackV2(
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"want " + child.toObjectId().getName() + "\n",
+			"deepen 1\n",
+			"done\n",
+			PacketLineIn.END);
+		PacketLineIn pckIn = new PacketLineIn(recvStream);
+		assertThat(pckIn.readString(), is("shallow-info"));
+		assertThat(pckIn.readString(), is("shallow " + child.toObjectId().getName()));
+		assertThat(pckIn.readString(), theInstance(PacketLineIn.DELIM));
+		assertThat(pckIn.readString(), is("packfile"));
+		parsePack(recvStream);
+		assertTrue(client.hasObject(child.toObjectId()));
+		assertFalse(client.hasObject(parent.toObjectId()));
+
+		// Without that, the parent is sent too.
+		recvStream = uploadPackV2(
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"want " + child.toObjectId().getName() + "\n",
+			"done\n",
+			PacketLineIn.END);
+		pckIn = new PacketLineIn(recvStream);
+		assertThat(pckIn.readString(), is("packfile"));
+		parsePack(recvStream);
+		assertTrue(client.hasObject(parent.toObjectId()));
+	}
+
+	@Test
+	public void testV2FetchDeepenWithoutDone() throws Exception {
+		RevCommit parent = remote.commit().message("parent").create();
+		RevCommit child = remote.commit().message("x").parent(parent).create();
+		remote.update("branch1", child);
+
+		ByteArrayInputStream recvStream = uploadPackV2(
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"want " + child.toObjectId().getName() + "\n",
+			"deepen 1\n",
+			PacketLineIn.END);
+		PacketLineIn pckIn = new PacketLineIn(recvStream);
+
+		// Verify that only the correct section is sent. "shallow-info"
+		// is not sent because, according to the specification, it is
+		// sent only if a packfile is sent.
+		assertThat(pckIn.readString(), is("acknowledgments"));
+		assertThat(pckIn.readString(), is("NAK"));
+		assertThat(pckIn.readString(), theInstance(PacketLineIn.END));
+	}
+
 	private static class RejectAllRefFilter implements RefFilter {
 		@Override
 		public Map<String, Ref> filter(Map<String, Ref> refs) {

org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties

@@ -224,6 +224,8 @@ credentialPassword=Password
 credentialUsername=Username
 daemonAlreadyRunning=Daemon already running
 daysAgo={0} days ago
+deepenNotWithDeepen=Cannot combine deepen with deepen-not
+deepenSinceWithDeepen=Cannot combine deepen with deepen-since
 deleteBranchUnexpectedResult=Delete branch returned unexpected result {0}
 deleteFileFailed=Could not delete file {0}
 deleteRequiresZeroNewId=Delete requires new ID to be zero
@@ -395,6 +397,7 @@ invalidStageForPath=Invalid stage {0} for path {1}
 invalidSystemProperty=Invalid system property ''{0}'': ''{1}''; using default value {2}
 invalidTagOption=Invalid tag option: {0}
 invalidTimeout=Invalid timeout: {0}
+invalidTimestamp=Invalid timestamp in {0}
 invalidTimeUnitValue2=Invalid time unit value: {0}.{1}={2}
 invalidTimeUnitValue3=Invalid time unit value: {0}.{1}.{2}={3}
 invalidTreeZeroLengthName=Cannot append a tree entry with zero-length name

org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java

@@ -285,6 +285,8 @@ public static JGitText get() {
 	/***/ public String credentialUsername;
 	/***/ public String daemonAlreadyRunning;
 	/***/ public String daysAgo;
+	/***/ public String deepenNotWithDeepen;
+	/***/ public String deepenSinceWithDeepen;
 	/***/ public String deleteBranchUnexpectedResult;
 	/***/ public String deleteFileFailed;
 	/***/ public String deleteRequiresZeroNewId;
@@ -455,6 +457,7 @@ public static JGitText get() {
 	/***/ public String invalidSystemProperty;
 	/***/ public String invalidTagOption;
 	/***/ public String invalidTimeout;
+	/***/ public String invalidTimestamp;
 	/***/ public String invalidTimeUnitValue2;
 	/***/ public String invalidTimeUnitValue3;
 	/***/ public String invalidTreeZeroLengthName;

org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java

@@ -107,6 +107,14 @@ public class GitProtocolConstants {
 	 */
 	public static final String OPTION_SHALLOW = "shallow"; //$NON-NLS-1$
 
+	/**
+	 * The client wants the "deepen" command to be interpreted as relative to
+	 * the client's shallow commits.
+	 *
+	 * @since 5.0
+	 */
+	public static final String OPTION_DEEPEN_RELATIVE = "deepen-relative"; //$NON-NLS-1$
+
 	/**
 	 * The client does not want progress messages and will ignore them.
 	 *

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -50,6 +50,7 @@
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
+import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_DEEPEN_RELATIVE;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_FILTER;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG;
 import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK;
@@ -123,7 +124,7 @@ public class UploadPack {
 	private static final String[] v2CapabilityAdvertisement = {
 		"version 2", //$NON-NLS-1$
 		COMMAND_LS_REFS,
-		COMMAND_FETCH
+		COMMAND_FETCH + '=' + OPTION_SHALLOW
 	};
 
 	/** Policy the server uses to validate client requests */
@@ -302,6 +303,19 @@ public Set<String> getOptions() {
 	/** Desired depth from the client on a shallow request. */
 	private int depth;
 
+	/**
+	 * Commit time of the newest objects the client has asked us using
+	 * --shallow-since not to send. Cannot be nonzero if depth is nonzero.
+	 */
+	private int shallowSince;
+
+	/**
+	 * (Possibly short) ref names, ancestors of which the client has asked us
+	 * not to send using --shallow-exclude. Cannot be non-null if depth is
+	 * nonzero.
+	 */
+	private @Nullable List<String> shallowExcludeRefs;
+
 	/** Commit time of the oldest common commit, in seconds. */
 	private int oldestTime;
 
@@ -813,7 +827,7 @@ else if (requestValidator instanceof AnyRequestValidator)
 			if (!clientShallowCommits.isEmpty())
 				verifyClientShallow();
 			if (depth != 0)
-				processShallow(unshallowCommits);
+				processShallow(null, unshallowCommits, true);
 			if (!clientShallowCommits.isEmpty())
 				walk.assumeShallow(clientShallowCommits);
 			sendPack = negotiate(accumulator);
@@ -968,12 +982,65 @@ private void fetchV2() throws IOException {
 				includeTag = true;
 			} else if (line.equals(OPTION_OFS_DELTA)) {
 				options.add(OPTION_OFS_DELTA);
+			} else if (line.startsWith("shallow ")) { //$NON-NLS-1$
+				clientShallowCommits.add(ObjectId.fromString(line.substring(8)));
+			} else if (line.startsWith("deepen ")) { //$NON-NLS-1$
+				depth = Integer.parseInt(line.substring(7));
+				if (depth <= 0) {
+					throw new PackProtocolException(
+							MessageFormat.format(JGitText.get().invalidDepth,
+									Integer.valueOf(depth)));
+				}
+				if (shallowSince != 0) {
+					throw new PackProtocolException(
+							JGitText.get().deepenSinceWithDeepen);
+				}
+				if (shallowExcludeRefs != null) {
+					throw new PackProtocolException(
+							JGitText.get().deepenNotWithDeepen);
+				}
+			} else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$
+				List<String> exclude = shallowExcludeRefs;
+				if (exclude == null) {
+					exclude = shallowExcludeRefs = new ArrayList<>();
+				}
+				exclude.add(line.substring(11));
+				if (depth != 0) {
+					throw new PackProtocolException(
+							JGitText.get().deepenNotWithDeepen);
+				}
+			} else if (line.equals(OPTION_DEEPEN_RELATIVE)) {
+				options.add(OPTION_DEEPEN_RELATIVE);
+			} else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$
+				shallowSince = Integer.parseInt(line.substring(13));
+				if (shallowSince <= 0) {
+					throw new PackProtocolException(
+							MessageFormat.format(
+									JGitText.get().invalidTimestamp, line));
+				}
+				if (depth !=  0) {
+					throw new PackProtocolException(
+							JGitText.get().deepenSinceWithDeepen);
+				}
 			}
 			// else ignore it
 		}
 		rawOut.stopBuffering();
 
 		boolean sectionSent = false;
+		@Nullable List<ObjectId> shallowCommits = null;
+		List<ObjectId> unshallowCommits = new ArrayList<>();
+
+		if (!clientShallowCommits.isEmpty()) {
+			verifyClientShallow();
+		}
+		if (depth != 0 || shallowSince != 0 || shallowExcludeRefs != null) {
+			shallowCommits = new ArrayList<ObjectId>();
+			processShallow(shallowCommits, unshallowCommits, false);
+		}
+		if (!clientShallowCommits.isEmpty())
+			walk.assumeShallow(clientShallowCommits);
+
 		if (doneReceived) {
 			processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE));
 		} else {
@@ -991,7 +1058,21 @@ private void fetchV2() throws IOException {
 			}
 			sectionSent = true;
 		}
+
 		if (doneReceived || okToGiveUp()) {
+			if (shallowCommits != null) {
+				if (sectionSent)
+					pckOut.writeDelim();
+				pckOut.writeString("shallow-info\n"); //$NON-NLS-1$
+				for (ObjectId o : shallowCommits) {
+					pckOut.writeString("shallow " + o.getName() + '\n'); //$NON-NLS-1$
+				}
+				for (ObjectId o : unshallowCommits) {
+					pckOut.writeString("unshallow " + o.getName() + '\n'); //$NON-NLS-1$
+				}
+				sectionSent = true;
+			}
+
 			if (sectionSent)
 				pckOut.writeDelim();
 			pckOut.writeString("packfile\n"); //$NON-NLS-1$
@@ -1078,9 +1159,21 @@ private static Set<ObjectId> refIdSet(Collection<Ref> refs) {
 
 	/*
 	 * Determines what "shallow" and "unshallow" lines to send to the user.
-	 * The information is written to pckOut and unshallowCommits.
+	 * The information is written to shallowCommits (if not null) and
+	 * unshallowCommits, and also written to #pckOut (if writeToPckOut is
+	 * true).
 	 */
-	private void processShallow(List<ObjectId> unshallowCommits) throws IOException {
+	private void processShallow(@Nullable List<ObjectId> shallowCommits,
+			List<ObjectId> unshallowCommits,
+			boolean writeToPckOut) throws IOException {
+		if (options.contains(OPTION_DEEPEN_RELATIVE) ||
+				shallowSince != 0 ||
+				shallowExcludeRefs != null) {
+			// TODO(jonathantanmy): Implement deepen-relative, deepen-since,
+			// and deepen-not.
+			throw new UnsupportedOperationException();
+		}
+
 		int walkDepth = depth - 1;
 		try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk(
 				walk.getObjectReader(), walkDepth)) {
@@ -1101,19 +1194,29 @@ private void processShallow(List<ObjectId> unshallowCommits) throws IOException
 				// Commits at the boundary which aren't already shallow in
 				// the client need to be marked as such
 				if (c.getDepth() == walkDepth
-						&& !clientShallowCommits.contains(c))
+						&& !clientShallowCommits.contains(c)) {
-					pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
+					if (shallowCommits != null) {
+						shallowCommits.add(c.copy());
+					}
+					if (writeToPckOut) {
+						pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
+					}
+				}
 
 				// Commits not on the boundary which are shallow in the client
 				// need to become unshallowed
 				if (c.getDepth() < walkDepth
 						&& clientShallowCommits.remove(c)) {
 					unshallowCommits.add(c.copy());
-					pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
+					if (writeToPckOut) {
+						pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
+					}
 				}
 			}
 		}
-		pckOut.end();
+		if (writeToPckOut) {
+			pckOut.end();
+		}
 	}
 
 	private void verifyClientShallow()