[pgm] Fetch-CLI: add support for shallow

This adds support for shallow cloning. The CloneCommand and the
FetchCommand now have the new options --depth, --shallow-since and
--shallow-exclude to tell the server that the client doesn't want to
download the complete history.

Bug: https://bugs.eclipse.org/bugs/show_bug.cgi?id=475615
Change-Id: I8f113bed25dd6df64f2f95de6a59d4675ab8a903

org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java

@@ -17,14 +17,20 @@
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
+import java.time.Instant;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.MockSystemReader;
+import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.lib.CLIRepositoryTestCase;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.RefUpdate;
 import org.eclipse.jgit.lib.Repository;
@@ -41,10 +47,14 @@ public class CloneTest extends CLIRepositoryTestCase {
 
 	private Git git;
 
+	private TestRepository<Repository> tr;
+
 	@Override
 	@Before
 	public void setUp() throws Exception {
 		super.setUp();
+
+		tr = new TestRepository<>(db);
 		git = new Git(db);
 	}
 
@@ -112,6 +122,22 @@ private RevCommit createInitialCommit() throws Exception {
 		return git.commit().setMessage("Initial commit").call();
 	}
 
+	private RevCommit createSecondCommit() throws Exception {
+		JGitTestUtil.writeTrashFile(db, "Test.txt", "Some change");
+		git.add().addFilepattern("Test.txt").call();
+		return git.commit()
+				.setCommitter(new PersonIdent(this.committer, tr.getDate()))
+				.setMessage("Second commit").call();
+	}
+
+	private RevCommit createThirdCommit() throws Exception {
+		JGitTestUtil.writeTrashFile(db, "change.txt", "another change");
+		git.add().addFilepattern("change.txt").call();
+		return git.commit()
+				.setCommitter(new PersonIdent(this.committer, tr.getDate()))
+				.setMessage("Third commit").call();
+	}
+
 	@Test
 	public void testCloneEmpty() throws Exception {
 		File gitDir = db.getDirectory();
@@ -203,4 +229,117 @@ public void testCloneMirror() throws Exception {
 		assertEquals("refs/*", fetchRefSpec.getDestination());
 		assertNotNull(git2.getRepository().exactRef("refs/meta/foo/bar"));
 	}
+
+	@Test
+	public void testDepth() throws Exception {
+		createInitialCommit();
+		createSecondCommit();
+		createThirdCommit();
+
+		File gitDir = db.getDirectory();
+		String sourceURI = gitDir.toURI().toString();
+		File target = createTempDirectory("target");
+		String cmd = "git clone --depth 1 " + sourceURI + " "
+				+ shellQuote(target.getPath());
+		String[] result = execute(cmd);
+		assertArrayEquals(new String[] {
+				"Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+		Git git2 = Git.open(target);
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(1, log.size());
+		RevCommit commit = log.get(0);
+		assertEquals(Set.of(commit.getId()),
+				git2.getRepository().getObjectDatabase().getShallowCommits());
+		assertEquals("Third commit", commit.getFullMessage());
+		assertEquals(0, commit.getParentCount());
+	}
+
+	@Test
+	public void testDepth2() throws Exception {
+		createInitialCommit();
+		createSecondCommit();
+		createThirdCommit();
+
+		File gitDir = db.getDirectory();
+		String sourceURI = gitDir.toURI().toString();
+		File target = createTempDirectory("target");
+		String cmd = "git clone --depth 2 " + sourceURI + " "
+				+ shellQuote(target.getPath());
+		String[] result = execute(cmd);
+		assertArrayEquals(new String[] {
+				"Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+		Git git2 = Git.open(target);
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(List.of("Third commit", "Second commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+	}
+
+	@Test
+	public void testCloneRepositoryWithShallowSince() throws Exception {
+		createInitialCommit();
+		tr.tick(30);
+		RevCommit secondCommit = createSecondCommit();
+		tr.tick(45);
+		createThirdCommit();
+
+		File gitDir = db.getDirectory();
+		String sourceURI = gitDir.toURI().toString();
+		File target = createTempDirectory("target");
+		String cmd = "git clone --shallow-since="
+				+ Instant.ofEpochSecond(secondCommit.getCommitTime()).toString()
+				+ " " + sourceURI + " " + shellQuote(target.getPath());
+		String[] result = execute(cmd);
+		assertArrayEquals(new String[] {
+				"Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+		Git git2 = Git.open(target);
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(2, log.size());
+		assertEquals(List.of("Third commit", "Second commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+	}
+
+	@Test
+	public void testCloneRepositoryWithShallowExclude() throws Exception {
+		final RevCommit firstCommit = createInitialCommit();
+		final RevCommit secondCommit = createSecondCommit();
+		createThirdCommit();
+
+		File gitDir = db.getDirectory();
+		String sourceURI = gitDir.toURI().toString();
+		File target = createTempDirectory("target");
+		String cmd = "git clone --shallow-exclude="
+				+ firstCommit.getId().getName() + " --shallow-exclude="
+				+ secondCommit.getId().getName() + " " + sourceURI + " "
+				+ shellQuote(target.getPath());
+		String[] result = execute(cmd);
+		assertArrayEquals(new String[] {
+				"Cloning into '" + target.getPath() + "'...", "", "" }, result);
+
+		Git git2 = Git.open(target);
+		addRepoToClose(git2.getRepository());
+
+		List<RevCommit> log = StreamSupport
+				.stream(git2.log().all().call().spliterator(), false)
+				.collect(Collectors.toList());
+		assertEquals(1, log.size());
+		assertEquals(List.of("Third commit"), log.stream()
+				.map(RevCommit::getFullMessage).collect(Collectors.toList()));
+	}
+
 }

org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties

@@ -137,6 +137,7 @@ metaVar_commitOrTag=COMMIT|TAG
 metaVar_commitPaths=paths
 metaVar_configFile=FILE
 metaVar_connProp=conn.prop
+metaVar_depth=<depth>
 metaVar_diffAlg=ALGORITHM
 metaVar_directory=DIRECTORY
 metaVar_extraArgument=ours|theirs
@@ -144,6 +145,7 @@ metaVar_file=FILE
 metaVar_filepattern=filepattern
 metaVar_gitDir=GIT_DIR
 metaVar_hostName=HOSTNAME
+metaVar_instant=<instant>
 metaVar_lfsStorage=STORAGE
 metaVar_linesOfContext=lines
 metaVar_message=message
@@ -168,6 +170,8 @@ metaVar_s3Region=REGION
 metaVar_s3StorageClass=STORAGE-CLASS
 metaVar_seconds=SECONDS
 metaVar_service=SERVICE
+metaVar_shallowExclude=<revision>
+metaVar_shallowSince=<date>
 metaVar_tagLocalUser=<GPG key ID>
 metaVar_tool=TOOL
 metaVar_treeish=tree-ish
@@ -374,6 +378,7 @@ usage_detectRenames=detect renamed files
 usage_diffAlgorithm=the diff algorithm to use. Currently supported are: 'myers', 'histogram'
 usage_DiffTool=git difftool is a Git command that allows you to compare and edit files between revisions using common diff tools.\ngit difftool is a frontend to git diff and accepts the same options and arguments.
 usage_MergeTool=git-mergetool - Run merge conflict resolution tools to resolve merge conflicts.\nUse git mergetool to run one of several merge utilities to resolve merge conflicts. It is typically run after git merge.
+usage_depth=Limit fetching to the specified number of commits from the tip of each remote branch history.
 usage_directoriesToExport=directories to export
 usage_disableTheServiceInAllRepositories=disable the service in all repositories
 usage_displayAListOfAllRegisteredJgitCommands=Display a list of all registered jgit commands
@@ -447,6 +452,8 @@ usage_resetMixed=Resets the index but not the working tree
 usage_runLfsStore=Run LFS Store in a given directory
 usage_S3NoSslVerify=Skip verification of Amazon server certificate and hostname
 usage_setTheGitRepositoryToOperateOn=set the git repository to operate on
+usage_shallowExclude=Deepen or shorten the history of a shallow repository to exclude commits reachable from a specified remote branch or tag. 
+usage_shallowSince=Deepen or shorten the history of a shallow repository to include all reachable commits after <date>.
 usage_show=Display one commit
 usage_showRefNamesMatchingCommits=Show ref names matching commits
 usage_showPatch=display patch

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java

@@ -13,7 +13,10 @@
 import java.io.File;
 import java.io.IOException;
 import java.text.MessageFormat;
+import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import org.eclipse.jgit.api.CloneCommand;
 import org.eclipse.jgit.api.Git;
@@ -48,6 +51,15 @@ class Clone extends AbstractFetchCommand implements CloneCommand.Callback {
 	@Option(name = "--quiet", usage = "usage_quiet")
 	private Boolean quiet;
 
+	@Option(name = "--depth", metaVar = "metaVar_depth", usage = "usage_depth")
+	private Integer depth = null;
+
+	@Option(name = "--shallow-since", metaVar = "metaVar_shallowSince", usage = "usage_shallowSince")
+	private Instant shallowSince = null;
+
+	@Option(name = "--shallow-exclude", metaVar = "metaVar_shallowExclude", usage = "usage_shallowExclude")
+	private List<String> shallowExcludes = new ArrayList<>();
+
 	@Option(name = "--recurse-submodules", usage = "usage_recurseSubmodules")
 	private boolean cloneSubmodules;
 
@@ -97,6 +109,16 @@ protected void run() throws Exception {
 				.setMirror(isMirror).setNoCheckout(noCheckout).setBranch(branch)
 				.setCloneSubmodules(cloneSubmodules).setTimeout(timeout);
 
+		if (depth != null) {
+			command.setDepth(depth.intValue());
+		}
+		if (shallowSince != null) {
+			command.setShallowSince(shallowSince);
+		}
+		for (String shallowExclude : shallowExcludes) {
+			command.addShallowExclude(shallowExclude);
+		}
+
 		command.setGitDir(gitdir == null ? null : new File(gitdir));
 		command.setDirectory(localNameF);
 		boolean msgs = quiet == null || !quiet.booleanValue();

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Fetch.java

@@ -14,6 +14,8 @@
 
 import java.io.IOException;
 import java.text.MessageFormat;
+import java.time.Instant;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.jgit.api.FetchCommand;
@@ -62,6 +64,15 @@ void nothin(@SuppressWarnings("unused") final boolean ignored) {
 	@Option(name = "--tags", usage="usage_tags", aliases = { "-t" })
 	private Boolean tags;
 
+	@Option(name = "--depth", metaVar = "metaVar_depth", usage = "usage_depth")
+	private Integer depth = null;
+
+	@Option(name = "--shallow-since", metaVar = "metaVar_shallowSince", usage = "usage_shallowSince")
+	private Instant shallowSince = null;
+
+	@Option(name = "--shallow-exclude", metaVar = "metaVar_shallowExclude", usage = "usage_shallowExclude")
+	private List<String> shallowExcludes = new ArrayList<>();
+
 	@Option(name = "--no-tags", usage = "usage_notags", aliases = { "-n" })
 	void notags(@SuppressWarnings("unused")
 	final boolean ignored) {
@@ -120,6 +131,15 @@ protected void run() {
 				fetch.setTagOpt(tags.booleanValue() ? TagOpt.FETCH_TAGS
 						: TagOpt.NO_TAGS);
 			}
+			if (depth != null) {
+				fetch.setDepth(depth.intValue());
+			}
+			if (shallowSince != null) {
+				fetch.setShallowSince(shallowSince);
+			}
+			for (String shallowExclude : shallowExcludes) {
+				fetch.addShallowExclude(shallowExclude);
+			}
 			if (0 <= timeout) {
 				fetch.setTimeout(timeout);
 			}

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java

@@ -214,6 +214,7 @@ public static String fatalError(String message) {
 	/***/ public String metaVar_filepattern;
 	/***/ public String metaVar_gitDir;
 	/***/ public String metaVar_hostName;
+	/***/ public String metaVar_instant;
 	/***/ public String metaVar_lfsStorage;
 	/***/ public String metaVar_linesOfContext;
 	/***/ public String metaVar_message;

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/CmdLineParser.java

@@ -13,6 +13,7 @@
 import java.io.IOException;
 import java.io.Writer;
 import java.lang.reflect.Field;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -55,6 +56,7 @@ public class CmdLineParser extends org.kohsuke.args4j.CmdLineParser {
 		registry.registerHandler(RevCommit.class, RevCommitHandler.class);
 		registry.registerHandler(RevTree.class, RevTreeHandler.class);
 		registry.registerHandler(List.class, OptionWithValuesListHandler.class);
+		registry.registerHandler(Instant.class, InstantHandler.class);
 	}
 
 	private final Repository db;

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/opt/InstantHandler.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2022, Harald Weiner and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v. 1.0 which is available at
+ * https://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+package org.eclipse.jgit.pgm.opt;
+
+import java.time.Instant;
+
+import org.eclipse.jgit.pgm.internal.CLIText;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.OptionDef;
+import org.kohsuke.args4j.spi.OptionHandler;
+import org.kohsuke.args4j.spi.Parameters;
+import org.kohsuke.args4j.spi.Setter;
+
+/**
+ * Custom argument handler {@link java.time.Instant} from string values.
+ * <p>
+ * Assumes the parser has been initialized with a Repository.
+ *
+ * @since 6.5
+ */
+public class InstantHandler extends OptionHandler<Instant> {
+	/**
+	 * Create a new handler for the command name.
+	 * <p>
+	 * This constructor is used only by args4j.
+	 *
+	 * @param parser
+	 *            a {@link org.kohsuke.args4j.CmdLineParser} object.
+	 * @param option
+	 *            a {@link org.kohsuke.args4j.OptionDef} object.
+	 * @param setter
+	 *            a {@link org.kohsuke.args4j.spi.Setter} object.
+	 */
+	public InstantHandler(CmdLineParser parser, OptionDef option,
+			Setter<? super Instant> setter) {
+		super(parser, option, setter);
+	}
+
+	/** {@inheritDoc} */
+	@Override
+	public int parseArguments(Parameters params) throws CmdLineException {
+		Instant instant = Instant.parse(params.getParameter(0));
+		setter.addValue(instant);
+		return 1;
+	}
+
+	/** {@inheritDoc} */
+	@Override
+	public String getDefaultMetaVariable() {
+		return CLIText.get().metaVar_instant;
+	}
+}