UploadPack: support deepen-since in protocol v2

Support the deepen-since parameter when requested by a client using
protocol v2. This is done by:
 - adding a DepthWalk.RevWalk#setDeepenSince method
 - updating DepthGenerator to recognize when deepen-since is set
 - recording in DepthWalk.Commit whether a commit is a boundary commit

Existing users of DepthWalk such as UploadPack previously recognized
boundary commits by comparing their depths against the threshold, not
tracking whether any parents were truly excluded. This behavior is
preserved - UploadPack considers a commit as boundary if its depth is
equal to the threshold *or* a parent was excluded (whether by depth or
by deepen-since).

Change-Id: I852bba6b1279f9cc8aee38282e9339d62b8dcddc
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java

@@ -27,6 +27,7 @@
 import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.NullProgressMonitor;
+import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
@@ -1190,6 +1191,59 @@ public void testV2FetchDeepenWithoutDone() throws Exception {
 		assertThat(pckIn.readString(), theInstance(PacketLineIn.END));
 	}
 
+	@Test
+	public void testV2FetchShallowSince() throws Exception {
+		PersonIdent person = new PersonIdent(remote.getRepository());
+
+		RevCommit beyondBoundary = remote.commit()
+			.committer(new PersonIdent(person, 1510000000, 0)).create();
+		RevCommit boundary = remote.commit().parent(beyondBoundary)
+			.committer(new PersonIdent(person, 1520000000, 0)).create();
+		RevCommit tooOld = remote.commit()
+			.committer(new PersonIdent(person, 1500000000, 0)).create();
+		RevCommit merge = remote.commit().parent(boundary).parent(tooOld)
+			.committer(new PersonIdent(person, 1530000000, 0)).create();
+
+		remote.update("branch1", merge);
+
+		// Report that we only have "boundary" as a shallow boundary.
+		ByteArrayInputStream recvStream = uploadPackV2(
+			"command=fetch\n",
+			PacketLineIn.DELIM,
+			"shallow " + boundary.toObjectId().getName() + "\n",
+			"deepen-since 1510000\n",
+			"want " + merge.toObjectId().getName() + "\n",
+			"have " + boundary.toObjectId().getName() + "\n",
+			"done\n",
+			PacketLineIn.END);
+		PacketLineIn pckIn = new PacketLineIn(recvStream);
+		assertThat(pckIn.readString(), is("shallow-info"));
+
+		// "merge" is shallow because one of its parents is committed
+		// earlier than the given deepen-since time.
+		assertThat(pckIn.readString(), is("shallow " + merge.toObjectId().getName()));
+
+		// "boundary" is unshallow because its parent committed at or
+		// later than the given deepen-since time.
+		assertThat(pckIn.readString(), is("unshallow " + boundary.toObjectId().getName()));
+
+		assertThat(pckIn.readString(), theInstance(PacketLineIn.DELIM));
+		assertThat(pckIn.readString(), is("packfile"));
+		parsePack(recvStream);
+
+		// The server does not send this because it is committed
+		// earlier than the given deepen-since time.
+		assertFalse(client.hasObject(tooOld.toObjectId()));
+
+		// The server does not send this because the client claims to
+		// have it.
+		assertFalse(client.hasObject(boundary.toObjectId()));
+
+		// The server sends both these commits.
+		assertTrue(client.hasObject(beyondBoundary.toObjectId()));
+		assertTrue(client.hasObject(merge.toObjectId()));
+	}
+
 	@Test
 	public void testV2FetchUnrecognizedArgument() throws Exception {
 		thrown.expect(PackProtocolException.class);

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DepthGenerator.java

@@ -59,6 +59,8 @@ class DepthGenerator extends Generator {
 
 	private final int depth;
 
+	private final int deepenSince;
+
 	private final RevWalk walk;
 
 	/**
@@ -91,6 +93,7 @@ class DepthGenerator extends Generator {
 		walk = (RevWalk)w;
 
 		this.depth = w.getDepth();
+		this.deepenSince = w.getDeepenSince();
 		this.UNSHALLOW = w.getUnshallowFlag();
 		this.REINTERESTING = w.getReinterestingFlag();
 
@@ -142,12 +145,24 @@ RevCommit next() throws MissingObjectException,
 				// this depth is guaranteed to be the smallest value that
 				// any path could produce.
 				if (dp.depth == -1) {
+					boolean failsDeepenSince = false;
+					if (deepenSince != 0) {
+						if ((p.flags & RevWalk.PARSED) == 0) {
+							p.parseHeaders(walk);
+						}
+						failsDeepenSince =
+							p.getCommitTime() < deepenSince;
+					}
+
 					dp.depth = newDepth;
 
 					// If the parent is not too deep, add it to the queue
 					// so that we can produce it later
-					if (newDepth <= depth)
+					if (newDepth <= depth && !failsDeepenSince) {
 						pending.add(p);
+					} else {
+						c.isBoundary = true;
+					}
 				}
 
 				// If the current commit has become unshallowed, everything

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DepthWalk.java

@@ -63,6 +63,15 @@ public interface DepthWalk {
 	 */
 	public int getDepth();
 
+	/**
+	 * @return the deepen-since value; if not 0, this walk only returns commits
+	 *         whose commit time is at or after this limit
+	 * @since 5.2
+	 */
+	public default int getDeepenSince() {
+		return 0;
+	}
+
 	/** @return flag marking commits that should become unshallow. */
 	/**
 	 * Get flag marking commits that should become unshallow.
@@ -83,11 +92,22 @@ public static class Commit extends RevCommit {
 		/** Depth of this commit in the graph, via shortest path. */
 		int depth;
 
+		boolean isBoundary;
+
 		/** @return depth of this commit, as found by the shortest path. */
 		public int getDepth() {
 			return depth;
 		}
 
+		/**
+		 * @return true if at least one of this commit's children was excluded
+		 *         due to a depth or shallow-since restriction, false otherwise
+		 * @since 5.2
+		 */
+		public boolean isBoundary() {
+			return isBoundary;
+		}
+
 		/**
 		 * Initialize a new commit.
 		 *
@@ -104,6 +124,8 @@ protected Commit(AnyObjectId id) {
 	public class RevWalk extends org.eclipse.jgit.revwalk.RevWalk implements DepthWalk {
 		private final int depth;
 
+		private int deepenSince;
+
 		private final RevFlag UNSHALLOW;
 
 		private final RevFlag REINTERESTING;
@@ -158,6 +180,22 @@ public int getDepth() {
 			return depth;
 		}
 
+		@Override
+		public int getDeepenSince() {
+			return deepenSince;
+		}
+
+		/**
+		 * Sets the deepen-since value.
+		 *
+		 * @param limit
+		 *            new deepen-since value
+		 * @since 5.2
+		 */
+		public void setDeepenSince(int limit) {
+			deepenSince = limit;
+		}
+
 		@Override
 		public RevFlag getUnshallowFlag() {
 			return UNSHALLOW;
@@ -174,6 +212,7 @@ public RevFlag getReinterestingFlag() {
 		@Override
 		public ObjectWalk toObjectWalkWithSameObjects() {
 			ObjectWalk ow = new ObjectWalk(reader, depth);
+			ow.deepenSince = deepenSince;
 			ow.objects = objects;
 			ow.freeFlags = freeFlags;
 			return ow;
@@ -184,6 +223,8 @@ public ObjectWalk toObjectWalkWithSameObjects() {
 	public class ObjectWalk extends org.eclipse.jgit.revwalk.ObjectWalk implements DepthWalk {
 		private final int depth;
 
+		private int deepenSince;
+
 		private final RevFlag UNSHALLOW;
 
 		private final RevFlag REINTERESTING;
@@ -262,6 +303,11 @@ public int getDepth() {
 			return depth;
 		}
 
+		@Override
+		public int getDeepenSince() {
+			return deepenSince;
+		}
+
 		@Override
 		public RevFlag getUnshallowFlag() {
 			return UNSHALLOW;

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -841,7 +841,7 @@ else if (requestValidator instanceof AnyRequestValidator)
 
 			if (!clientShallowCommits.isEmpty())
 				verifyClientShallow(clientShallowCommits);
-			if (depth != 0) {
+			if (depth != 0 || shallowSince != 0) {
 				computeShallowsAndUnshallows(wantIds, shallow -> {
 					pckOut.writeString("shallow " + shallow.name() + '\n'); //$NON-NLS-1$
 				}, unshallow -> {
@@ -1143,17 +1143,18 @@ private void computeShallowsAndUnshallows(Iterable<ObjectId> wants,
 			IOConsumer<ObjectId> shallowFunc,
 			IOConsumer<ObjectId> unshallowFunc)
 			throws IOException {
-		if (options.contains(OPTION_DEEPEN_RELATIVE) || shallowSince != 0
+		if (options.contains(OPTION_DEEPEN_RELATIVE) || !deepenNotRefs.isEmpty()) {
-				|| !deepenNotRefs.isEmpty()) {
+			// TODO(jonathantanmy): Implement deepen-relative
-			// TODO(jonathantanmy): Implement deepen-relative, deepen-since,
 			// and deepen-not.
 			throw new UnsupportedOperationException();
 		}
 
-		int walkDepth = depth - 1;
+		int walkDepth = depth == 0 ? Integer.MAX_VALUE : depth - 1;
 		try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk(
 				walk.getObjectReader(), walkDepth)) {
 
+			depthWalk.setDeepenSince(shallowSince);
+
 			// Find all the commits which will be shallow
 			for (ObjectId o : wants) {
 				try {
@@ -1167,17 +1168,17 @@ private void computeShallowsAndUnshallows(Iterable<ObjectId> wants,
 			while ((o = depthWalk.next()) != null) {
 				DepthWalk.Commit c = (DepthWalk.Commit) o;
 
+				boolean isBoundary = (c.getDepth() == walkDepth) || c.isBoundary();
+
 				// Commits at the boundary which aren't already shallow in
 				// the client need to be marked as such
-				if (c.getDepth() == walkDepth
+				if (isBoundary && !clientShallowCommits.contains(c)) {
-						&& !clientShallowCommits.contains(c)) {
 					shallowFunc.accept(c.copy());
 				}
 
 				// Commits not on the boundary which are shallow in the client
 				// need to become unshallowed
-				if (c.getDepth() < walkDepth
+				if (!isBoundary && clientShallowCommits.remove(c)) {
-						&& clientShallowCommits.remove(c)) {
 					unshallowFunc.accept(c.copy());
 				}
 			}
@@ -1987,9 +1988,11 @@ else if (ref.getName().startsWith(Constants.R_HEADS))
 			}
 
 			RevWalk rw = walk;
-			if (depth > 0) {
+			if (depth > 0 || shallowSince != 0) {
+				int walkDepth = depth == 0 ? Integer.MAX_VALUE : depth - 1;
 				pw.setShallowPack(depth, unshallowCommits);
-				rw = new DepthWalk.RevWalk(walk.getObjectReader(), depth - 1);
+				rw = new DepthWalk.RevWalk(walk.getObjectReader(), walkDepth);
+				((DepthWalk.RevWalk) rw).setDeepenSince(shallowSince);
 				rw.assumeShallow(clientShallowCommits);
 			}