diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java index 3438c52c8..6b067273a 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java @@ -24,13 +24,16 @@ import java.net.URI; import java.net.URL; import java.text.MessageFormat; +import java.time.Instant; import java.util.List; +import java.util.Set; import javax.servlet.http.HttpServletRequest; import org.eclipse.jetty.servlet.DefaultServlet; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jgit.api.Git; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.errors.TransportException; @@ -408,4 +411,110 @@ public void testV2HttpSubsequentResponse() throws Exception { assertEquals(200, c.getResponseCode()); } + + @Test + public void testCloneWithDepth() throws Exception { + remoteRepository.getRepository().getConfig().setInt( + "protocol", null, "version", 0); + File directory = createTempDirectory("testCloneWithDepth"); + Git git = Git.cloneRepository() + .setDirectory(directory) + .setDepth(1) + .setURI(smartAuthNoneURI.toString()) + .call(); + + assertEquals(Set.of(git.getRepository().resolve(Constants.HEAD)), git.getRepository().getObjectDatabase().getShallowCommits()); + } + + @Test + public void testCloneWithDeepenSince() throws Exception { + remoteRepository.getRepository().getConfig().setInt( + "protocol", null, "version", 0); + RevCommit commit = remoteRepository.commit() + .parent(remoteRepository.git().log().call().iterator().next()) + .message("Test") + .add("test.txt", "Hello world") + .create(); + remoteRepository.update(master, commit); + + File directory = createTempDirectory("testCloneWithDeepenSince"); + Git git = Git.cloneRepository() + .setDirectory(directory) + .setShallowSince(Instant.ofEpochSecond(commit.getCommitTime())) + .setURI(smartAuthNoneURI.toString()) + .call(); + + assertEquals(Set.of(git.getRepository().resolve(Constants.HEAD)), git.getRepository().getObjectDatabase().getShallowCommits()); + } + + @Test + public void testCloneWithDeepenNot() throws Exception { + remoteRepository.getRepository().getConfig().setInt( + "protocol", null, "version", 0); + RevCommit commit = remoteRepository.git().log().call().iterator().next(); + remoteRepository.update(master, remoteRepository.commit() + .parent(commit) + .message("Test") + .add("test.txt", "Hello world") + .create()); + + File directory = createTempDirectory("testCloneWithDeepenNot"); + Git git = Git.cloneRepository() + .setDirectory(directory) + .addShallowExclude(commit.getId()) + .setURI(smartAuthNoneURI.toString()) + .call(); + + assertEquals(Set.of(git.getRepository().resolve(Constants.HEAD)), git.getRepository().getObjectDatabase().getShallowCommits()); + } + + @Test + public void testV2CloneWithDepth() throws Exception { + File directory = createTempDirectory("testV2CloneWithDepth"); + Git git = Git.cloneRepository() + .setDirectory(directory) + .setDepth(1) + .setURI(smartAuthNoneURI.toString()) + .call(); + + assertEquals(Set.of(git.getRepository().resolve(Constants.HEAD)), git.getRepository().getObjectDatabase().getShallowCommits()); + } + + @Test + public void testV2CloneWithDeepenSince() throws Exception { + RevCommit commit = remoteRepository.commit() + .parent(remoteRepository.git().log().call().iterator().next()) + .message("Test") + .add("test.txt", "Hello world") + .create(); + remoteRepository.update(master, commit); + + File directory = createTempDirectory("testV2CloneWithDeepenSince"); + Git git = Git.cloneRepository() + .setDirectory(directory) + .setShallowSince(Instant.ofEpochSecond(commit.getCommitTime())) + .setURI(smartAuthNoneURI.toString()) + .call(); + + assertEquals(Set.of(git.getRepository().resolve(Constants.HEAD)), git.getRepository().getObjectDatabase().getShallowCommits()); + } + + @Test + public void testV2CloneWithDeepenNot() throws Exception { + RevCommit commit = remoteRepository.git().log().call().iterator().next(); + remoteRepository.update(master, remoteRepository.commit() + .parent(commit) + .message("Test") + .add("test.txt", "Hello world") + .create()); + + File directory = createTempDirectory("testV2CloneWithDeepenNot"); + Git git = Git.cloneRepository() + .setDirectory(directory) + .addShallowExclude(commit.getId()) + .setURI(smartAuthNoneURI.toString()) + .call(); + + assertEquals(Set.of(git.getRepository().resolve(Constants.HEAD)), git.getRepository().getObjectDatabase().getShallowCommits()); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java index c928d2ad2..6053c8c56 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2013 Chris Aniszczyk and others + * Copyright (C) 2011, 2022 Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -19,10 +19,14 @@ import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.time.Instant; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.eclipse.jgit.api.ListBranchCommand.ListMode; import org.eclipse.jgit.api.errors.GitAPIException; @@ -40,6 +44,7 @@ import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.submodule.SubmoduleStatus; import org.eclipse.jgit.submodule.SubmoduleStatusType; import org.eclipse.jgit.submodule.SubmoduleWalk; @@ -895,6 +900,234 @@ public void testCloneWithHeadSymRefIsNonMasterCopy() throws IOException, GitAPIE assertEquals("refs/heads/test-copy", git2.getRepository().getFullBranch()); } + @Test + public void testCloneRepositoryWithDepth() throws IOException, JGitInternalException, GitAPIException { + File directory = createTempDirectory("testCloneRepositoryWithDepth"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setDepth(1); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport.stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(1, log.size()); + RevCommit commit = log.get(0); + assertEquals(Set.of(commit.getId()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals("Second commit", commit.getFullMessage()); + assertEquals(0, commit.getParentCount()); + } + + @Test + public void testCloneRepositoryWithDepthAndAllBranches() throws IOException, JGitInternalException, GitAPIException { + File directory = createTempDirectory("testCloneRepositoryWithDepthAndAllBranches"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setDepth(1); + command.setCloneAllBranches(true); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport.stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals(log.stream().map(RevCommit::getId).collect(Collectors.toSet()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals(List.of("Second commit", "Initial commit"), + log.stream().map(RevCommit::getFullMessage).collect(Collectors.toList())); + for (RevCommit commit : log) { + assertEquals(0, commit.getParentCount()); + } + } + + @Test + public void testCloneRepositoryWithDepth2() throws Exception { + RevCommit parent = tr.git().log().call().iterator().next(); + RevCommit commit = tr.commit() + .parent(parent) + .message("Third commit") + .add("test.txt", "Hello world") + .create(); + tr.update("refs/heads/test", commit); + + File directory = createTempDirectory("testCloneRepositoryWithDepth2"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setDepth(2); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals(Set.of(parent.getId()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals(List.of("Third commit", "Second commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + assertEquals(List.of(Integer.valueOf(1), Integer.valueOf(0)), + log.stream().map(RevCommit::getParentCount) + .collect(Collectors.toList())); + } + + @Test + public void testCloneRepositoryWithDepthAndFetch() throws Exception { + File directory = createTempDirectory("testCloneRepositoryWithDepthAndFetch"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setDepth(1); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + RevCommit parent = tr.git().log().call().iterator().next(); + RevCommit commit = tr.commit() + .parent(parent) + .message("Third commit") + .add("test.txt", "Hello world") + .create(); + tr.update("refs/heads/test", commit); + + git2.fetch().call(); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals(Set.of(parent.getId()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals(List.of("Third commit", "Second commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + assertEquals(List.of(Integer.valueOf(1), Integer.valueOf(0)), + log.stream().map(RevCommit::getParentCount) + .collect(Collectors.toList())); + } + + @Test + public void testCloneRepositoryWithDepthAndFetchWithDepth() throws Exception { + File directory = createTempDirectory("testCloneRepositoryWithDepthAndFetchWithDepth"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setDepth(1); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + RevCommit parent = tr.git().log().call().iterator().next(); + RevCommit commit = tr.commit() + .parent(parent) + .message("Third commit") + .add("test.txt", "Hello world") + .create(); + tr.update("refs/heads/test", commit); + + git2.fetch().setDepth(1).call(); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals( + log.stream().map(RevObject::getId).collect(Collectors.toSet()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals(List.of("Third commit", "Second commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + assertEquals(List.of(Integer.valueOf(0), Integer.valueOf(0)), + log.stream().map(RevCommit::getParentCount) + .collect(Collectors.toList())); + } + + @Test + public void testCloneRepositoryWithDepthAndFetchUnshallow() throws Exception { + File directory = createTempDirectory("testCloneRepositoryWithDepthAndFetchUnshallow"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setDepth(1); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + git2.fetch().setUnshallow(true).call(); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(2, log.size()); + assertEquals(Set.of(), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals(List.of("Second commit", "Initial commit"), log.stream() + .map(RevCommit::getFullMessage).collect(Collectors.toList())); + assertEquals(List.of(Integer.valueOf(1), Integer.valueOf(0)), + log.stream().map(RevCommit::getParentCount) + .collect(Collectors.toList())); + } + + @Test + public void testCloneRepositoryWithShallowSince() throws Exception { + RevCommit commit = tr.commit() + .parent(tr.git().log().call().iterator().next()) + .message("Third commit").add("test.txt", "Hello world") + .create(); + tr.update("refs/heads/test", commit); + + File directory = createTempDirectory("testCloneRepositoryWithShallowSince"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.setShallowSince(Instant.ofEpochSecond(commit.getCommitTime())); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(1, log.size()); + assertEquals(Set.of(commit.getId()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals("Third commit", log.get(0).getFullMessage()); + assertEquals(0, log.get(0).getParentCount()); + } + + @Test + public void testCloneRepositoryWithShallowExclude() throws Exception { + RevCommit parent = tr.git().log().call().iterator().next(); + tr.update("refs/heads/test", + tr.commit() + .parent(parent) + .message("Third commit") + .add("test.txt", "Hello world") + .create()); + + File directory = createTempDirectory("testCloneRepositoryWithShallowExclude"); + CloneCommand command = Git.cloneRepository(); + command.setDirectory(directory); + command.setURI(fileUri()); + command.addShallowExclude(parent.getId()); + command.setBranchesToClone(Set.of("refs/heads/test")); + Git git2 = command.call(); + addRepoToClose(git2.getRepository()); + + List log = StreamSupport + .stream(git2.log().all().call().spliterator(), false) + .collect(Collectors.toList()); + assertEquals(1, log.size()); + RevCommit commit = log.get(0); + assertEquals(Set.of(commit.getId()), + git2.getRepository().getObjectDatabase().getShallowCommits()); + assertEquals("Third commit", commit.getFullMessage()); + assertEquals(0, commit.getParentCount()); + } + private void assertTagOption(Repository repo, TagOpt expectedTagOption) throws URISyntaxException { RemoteConfig remoteConfig = new RemoteConfig( diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java index c0db83a82..b2a4af30a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV0ParserTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -15,6 +15,7 @@ import static org.eclipse.jgit.lib.Constants.OBJ_TREE; import static org.eclipse.jgit.transport.ObjectIdMatcher.hasOnlyObjectIds; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -132,6 +133,42 @@ public void testRecvWantsDeepen() "f900c8326a43303685c46b279b9f70411bff1a4b")); } + @Test + public void testRecvWantsDeepenSince() + throws PackProtocolException, IOException { + PacketLineIn pckIn = formatAsPacketLine( + "want 4624442d68ee402a94364191085b77137618633e\n", + "want f900c8326a43303685c46b279b9f70411bff1a4b\n", + "deepen-since 1652773020\n", + PacketLineIn.end()); + ProtocolV0Parser parser = new ProtocolV0Parser(defaultConfig()); + FetchV0Request request = parser.recvWants(pckIn); + assertTrue(request.getClientCapabilities().isEmpty()); + assertEquals(1652773020, request.getDeepenSince()); + assertThat(request.getWantIds(), + hasOnlyObjectIds("4624442d68ee402a94364191085b77137618633e", + "f900c8326a43303685c46b279b9f70411bff1a4b")); + } + + @Test + public void testRecvWantsDeepenNots() + throws PackProtocolException, IOException { + PacketLineIn pckIn = formatAsPacketLine( + "want 4624442d68ee402a94364191085b77137618633e\n", + "want f900c8326a43303685c46b279b9f70411bff1a4b\n", + "deepen-not 856d5138d7269a483efe276d4a6b5c25b4fbb1a4\n", + "deepen-not heads/refs/test\n", + PacketLineIn.end()); + ProtocolV0Parser parser = new ProtocolV0Parser(defaultConfig()); + FetchV0Request request = parser.recvWants(pckIn); + assertTrue(request.getClientCapabilities().isEmpty()); + assertThat(request.getDeepenNots(), contains("856d5138d7269a483efe276d4a6b5c25b4fbb1a4", + "heads/refs/test")); + assertThat(request.getWantIds(), + hasOnlyObjectIds("4624442d68ee402a94364191085b77137618633e", + "f900c8326a43303685c46b279b9f70411bff1a4b")); + } + @Test public void testRecvWantsShallow() throws PackProtocolException, IOException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java index 837bdce91..167b5b72c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/ProtocolV2ParserTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -152,7 +152,7 @@ public void testFetchWithShallow_deepen() throws IOException { assertThat(request.getClientShallowCommits(), hasOnlyObjectIds("28274d02c489f4c7e68153056e9061a46f62d7a0", "145e683b229dcab9d0e2ccb01b386f9ecc17d29d")); - assertTrue(request.getDeepenNotRefs().isEmpty()); + assertTrue(request.getDeepenNots().isEmpty()); assertEquals(15, request.getDepth()); assertTrue(request.getClientCapabilities() .contains(GitProtocolConstants.OPTION_DEEPEN_RELATIVE)); @@ -171,7 +171,7 @@ public void testFetchWithShallow_deepenNot() throws IOException { assertThat(request.getClientShallowCommits(), hasOnlyObjectIds("28274d02c489f4c7e68153056e9061a46f62d7a0", "145e683b229dcab9d0e2ccb01b386f9ecc17d29d")); - assertThat(request.getDeepenNotRefs(), + assertThat(request.getDeepenNots(), hasItems("a08595f76159b09d57553e37a5123f1091bb13e7")); } diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 6eb8bd373..8aa84f3ac 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -13,7 +13,13 @@ - + + + + + + + @@ -60,14 +66,6 @@ - - - - - - - - diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 66adad515..84a7a80d6 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -237,6 +237,8 @@ deletedOrphanInPackDir=Deleted orphaned file {} deleteRequiresZeroNewId=Delete requires new ID to be zero deleteTagUnexpectedResult=Delete tag returned unexpected result {0} deletingNotSupported=Deleting {0} not supported. +depthMustBeAt1=Depth must be >= 1 +depthWithUnshallow=Depth and unshallow can\'t be used together destinationIsNotAWildcard=Destination is not a wildcard. detachedHeadDetected=HEAD is detached diffToolNotGivenError=No diff tool provided and no defaults configured. @@ -518,6 +520,7 @@ notFound=not found. nothingToFetch=Nothing to fetch. nothingToPush=Nothing to push. notMergedExceptionMessage=Branch was not deleted as it has not been merged yet; use the force option to delete it anyway +notShallowedUnshallow=The server sent a unshallow for a commit that wasn''t marked as shallow: {0} noXMLParserAvailable=No XML parser available. objectAtHasBadZlibStream=Object at {0} in {1} has bad zlib stream objectIsCorrupt=Object {0} is corrupt: {1} @@ -662,6 +665,7 @@ serviceNotEnabledNoName=Service not enabled serviceNotPermitted={1} not permitted on ''{0}'' sha1CollisionDetected=SHA-1 collision detected on {0} shallowCommitsAlreadyInitialized=Shallow commits have already been initialized +shallowNotSupported=The server does not support shallow shallowPacksRequireDepthWalk=Shallow packs require a DepthWalk shortCompressedStreamAt=Short compressed stream at {0} shortReadOfBlock=Short read of block. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java index 3aa711455..1f979a938 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2017 Chris Aniszczyk and others + * Copyright (C) 2011, 2022 Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,10 +13,13 @@ import java.io.IOException; import java.net.URISyntaxException; import java.text.MessageFormat; +import java.time.Instant; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidRemoteException; @@ -91,6 +94,12 @@ public class CloneCommand extends TransportCommand { private TagOpt tagOption; + private Integer depth; + + private Instant shallowSince; + + private List shallowExcludes = new ArrayList<>(); + private enum FETCH_TYPE { MULTIPLE_BRANCHES, ALL_BRANCHES, MIRROR } @@ -306,6 +315,11 @@ private FetchResult fetch(Repository clonedRepo, URIish u) fetchAll ? TagOpt.FETCH_TAGS : TagOpt.AUTO_FOLLOW); } command.setInitialBranch(branch); + if (depth != null) { + command.setDepth(depth.intValue()); + } + command.setShallowSince(shallowSince); + command.setShallowExcludes(shallowExcludes); configure(command); return command.call(); @@ -737,6 +751,82 @@ public CloneCommand setCallback(Callback callback) { return this; } + /** + * Creates a shallow clone with a history truncated to the specified number + * of commits. + * + * @param depth + * the depth + * @return {@code this} + * + * @since 6.3 + */ + public CloneCommand setDepth(int depth) { + if (depth < 1) { + throw new IllegalArgumentException(JGitText.get().depthMustBeAt1); + } + this.depth = Integer.valueOf(depth); + return this; + } + + /** + * Creates a shallow clone with a history after the specified time. + * + * @param shallowSince + * the timestammp; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public CloneCommand setShallowSince(@NonNull OffsetDateTime shallowSince) { + this.shallowSince = shallowSince.toInstant(); + return this; + } + + /** + * Creates a shallow clone with a history after the specified time. + * + * @param shallowSince + * the timestammp; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public CloneCommand setShallowSince(@NonNull Instant shallowSince) { + this.shallowSince = shallowSince; + return this; + } + + /** + * Creates a shallow clone with a history, excluding commits reachable from + * a specified remote branch or tag. + * + * @param shallowExclude + * the ref or commit; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public CloneCommand addShallowExclude(@NonNull String shallowExclude) { + shallowExcludes.add(shallowExclude); + return this; + } + + /** + * Creates a shallow clone with a history, excluding commits reachable from + * a specified remote branch or tag. + * + * @param shallowExclude + * the commit; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public CloneCommand addShallowExclude(@NonNull ObjectId shallowExclude) { + shallowExcludes.add(shallowExclude.name()); + return this; + } + private static void validateDirs(File directory, File gitDir, boolean bare) throws IllegalStateException { if (directory != null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java index 90c1515b0..84bee3620 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Chris Aniszczyk and others + * Copyright (C) 2010, 2022 Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -14,10 +14,13 @@ import java.io.IOException; import java.net.URISyntaxException; import java.text.MessageFormat; +import java.time.Instant; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidConfigurationException; @@ -76,6 +79,14 @@ public class FetchCommand extends TransportCommand { private String initialBranch; + private Integer depth; + + private Instant deepenSince; + + private List shallowExcludes = new ArrayList<>(); + + private boolean unshallow; + /** * Callback for status of fetch operation. * @@ -156,11 +167,9 @@ private void fetchSubmodules(FetchResult results) walk.getPath()); // When the fetch mode is "yes" we always fetch. When the - // mode - // is "on demand", we only fetch if the submodule's revision - // was - // updated to an object that is not currently present in the - // submodule. + // mode is "on demand", we only fetch if the submodule's + // revision was updated to an object that is not currently + // present in the submodule. if ((recurseMode == FetchRecurseSubmodulesMode.ON_DEMAND && !submoduleRepo.getObjectDatabase() .has(walk.getObjectId())) @@ -209,6 +218,17 @@ public FetchResult call() throws GitAPIException, InvalidRemoteException, if (tagOption != null) transport.setTagOpt(tagOption); transport.setFetchThin(thin); + if (depth != null) { + transport.setDepth(depth); + } + if (unshallow) { + if (depth != null) { + throw new IllegalStateException(JGitText.get().depthWithUnshallow); + } + transport.setDepth(Constants.INFINITE_DEPTH); + } + transport.setDeepenSince(deepenSince); + transport.setDeepenNots(shallowExcludes); configure(transport); FetchResult result = transport.fetch(monitor, applyOptions(refSpecs), initialBranch); @@ -542,4 +562,105 @@ public FetchCommand setForceUpdate(boolean force) { this.isForceUpdate = force; return this; } + + /** + * Limits fetching to the specified number of commits from the tip of each + * remote branch history. + * + * @param depth + * the depth + * @return {@code this} + * + * @since 6.3 + */ + public FetchCommand setDepth(int depth) { + if (depth < 1) { + throw new IllegalArgumentException(JGitText.get().depthMustBeAt1); + } + this.depth = Integer.valueOf(depth); + return this; + } + + /** + * Deepens or shortens the history of a shallow repository to include all + * reachable commits after a specified time. + * + * @param shallowSince + * the timestammp; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public FetchCommand setShallowSince(@NonNull OffsetDateTime shallowSince) { + this.deepenSince = shallowSince.toInstant(); + return this; + } + + /** + * Deepens or shortens the history of a shallow repository to include all + * reachable commits after a specified time. + * + * @param shallowSince + * the timestammp; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public FetchCommand setShallowSince(@NonNull Instant shallowSince) { + this.deepenSince = shallowSince; + return this; + } + + /** + * Deepens or shortens the history of a shallow repository to exclude + * commits reachable from a specified remote branch or tag. + * + * @param shallowExclude + * the ref or commit; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public FetchCommand addShallowExclude(@NonNull String shallowExclude) { + shallowExcludes.add(shallowExclude); + return this; + } + + /** + * Creates a shallow clone with a history, excluding commits reachable from + * a specified remote branch or tag. + * + * @param shallowExclude + * the commit; must not be {@code null} + * @return {@code this} + * + * @since 6.3 + */ + public FetchCommand addShallowExclude(@NonNull ObjectId shallowExclude) { + shallowExcludes.add(shallowExclude.name()); + return this; + } + + /** + * If the source repository is complete, converts a shallow repository to a + * complete one, removing all the limitations imposed by shallow + * repositories. + * + * If the source repository is shallow, fetches as much as possible so that + * the current repository has the same history as the source repository. + * + * @param unshallow + * whether to unshallow or not + * @return {@code this} + * + * @since 6.3 + */ + public FetchCommand setUnshallow(boolean unshallow) { + this.unshallow = unshallow; + return this; + } + + void setShallowExcludes(List shallowExcludes) { + this.shallowExcludes = shallowExcludes; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index efdb8e42e..551a5a8a9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -265,6 +265,8 @@ public static JGitText get() { /***/ public String deleteRequiresZeroNewId; /***/ public String deleteTagUnexpectedResult; /***/ public String deletingNotSupported; + /***/ public String depthMustBeAt1; + /***/ public String depthWithUnshallow; /***/ public String destinationIsNotAWildcard; /***/ public String detachedHeadDetected; /***/ public String diffToolNotGivenError; @@ -546,6 +548,7 @@ public static JGitText get() { /***/ public String nothingToFetch; /***/ public String nothingToPush; /***/ public String notMergedExceptionMessage; + /***/ public String notShallowedUnshallow; /***/ public String noXMLParserAvailable; /***/ public String objectAtHasBadZlibStream; /***/ public String objectIsCorrupt; @@ -690,6 +693,7 @@ public static JGitText get() { /***/ public String serviceNotPermitted; /***/ public String sha1CollisionDetected; /***/ public String shallowCommitsAlreadyInitialized; + /***/ public String shallowNotSupported; /***/ public String shallowPacksRequireDepthWalk; /***/ public String shortCompressedStreamAt; /***/ public String shortReadOfBlock; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java index 99da22239..5a8207ed0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java @@ -1,3 +1,12 @@ +/* + * Copyright (C) 2011, 2022 Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ package org.eclipse.jgit.internal.storage.dfs; import java.io.ByteArrayOutputStream; @@ -6,13 +15,16 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.RefDatabase; /** @@ -98,6 +110,7 @@ public void setGitwebDescription(@Nullable String d) { public static class MemObjDatabase extends DfsObjDatabase { private List packs = new ArrayList<>(); private int blockSize; + private Set shallowCommits = Collections.emptySet(); MemObjDatabase(DfsRepository repo) { super(repo, new DfsReaderOptions()); @@ -166,6 +179,16 @@ public void flush() { }; } + @Override + public Set getShallowCommits() throws IOException { + return shallowCommits; + } + + @Override + public void setShallowCommits(Set shallowCommits) { + this.shallowCommits = shallowCommits; + } + @Override public long getApproximateObjectCount() { long count = 0; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java index 094fdc155..9272bf3f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Constantine Plotnikov - * Copyright (C) 2010, JetBrains s.r.o. and others + * Copyright (C) 2010, 2022 JetBrains s.r.o. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -117,10 +117,15 @@ FS getFS() { } @Override - Set getShallowCommits() throws IOException { + public Set getShallowCommits() throws IOException { return wrapped.getShallowCommits(); } + @Override + public void setShallowCommits(Set shallowCommits) throws IOException { + wrapped.setShallowCommits(shallowCommits); + } + private CachedObjectDirectory[] myAlternates() { if (alts == null) { ObjectDirectory.AlternateHandle[] src = wrapped.myAlternates(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java index 01dd27d9f..e97ed393a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Google Inc. and others + * Copyright (C) 2010, 2022 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -50,8 +50,6 @@ abstract void resolve(Set matches, AbbreviatedObjectId id) abstract FS getFS(); - abstract Set getShallowCommits() throws IOException; - abstract void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 06c8cad3a..1a1d31a63 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, Google Inc. and others + * Copyright (C) 2009, 2022 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -19,6 +19,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; import java.text.MessageFormat; import java.util.ArrayList; @@ -560,7 +561,7 @@ FS getFS() { } @Override - Set getShallowCommits() throws IOException { + public Set getShallowCommits() throws IOException { if (shallowFile == null || !shallowFile.isFile()) return Collections.emptySet(); @@ -587,6 +588,43 @@ Set getShallowCommits() throws IOException { return shallowCommitsIds; } + @Override + public void setShallowCommits(Set shallowCommits) throws IOException { + this.shallowCommitsIds = shallowCommits; + LockFile lock = new LockFile(shallowFile); + if (!lock.lock()) { + throw new IOException(MessageFormat.format(JGitText.get().lockError, + shallowFile.getAbsolutePath())); + } + + try { + if (shallowCommits.isEmpty()) { + if (shallowFile.isFile()) { + shallowFile.delete(); + } + } else { + try (OutputStream out = lock.getOutputStream()) { + for (ObjectId shallowCommit : shallowCommits) { + byte[] buf = new byte[Constants.OBJECT_ID_STRING_LENGTH + 1]; + shallowCommit.copyTo(buf, 0); + buf[Constants.OBJECT_ID_STRING_LENGTH] = '\n'; + out.write(buf); + } + } finally { + lock.commit(); + } + } + } finally { + lock.unlock(); + } + + if (shallowCommits.isEmpty()) { + shallowFileSnapshot = FileSnapshot.DIRTY; + } else { + shallowFileSnapshot = FileSnapshot.save(shallowFile); + } + } + void closeAllPackHandles(File packFile) { // if the packfile already exists (because we are rewriting a // packfile for the same set of objects maybe with different diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index cf2e69dbb..30a007419 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2008, Google Inc. * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2006-2017, Shawn O. Pearce and others + * Copyright (C) 2006, 2022, Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -747,6 +747,13 @@ public static byte[] encode(String str) { */ public static final String LOCK_SUFFIX = ".lock"; //$NON-NLS-1$ + /** + * Depth used to unshallow a repository + * + * @since 6.3 + */ + public static final int INFINITE_DEPTH = 0x7fffffff; + private Constants() { // Hide the default constructor } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDatabase.java index 70009cba3..1c0f43609 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectDatabase.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, Google Inc. and others + * Copyright (C) 2009, 2022 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -11,6 +11,7 @@ package org.eclipse.jgit.lib; import java.io.IOException; +import java.util.Set; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; @@ -71,6 +72,26 @@ public void create() throws IOException { */ public abstract ObjectReader newReader(); + /** + * @return the shallow commits of the current repository + * + * @throws IOException the database could not be read + * + * @since 6.3 + */ + public abstract Set getShallowCommits() throws IOException; + + /** + * Update the shallow commits of the current repository + * + * @param shallowCommits the new shallow commits + * + * @throws IOException the database could not be updated + * + * @since 6.3 + */ + public abstract void setShallowCommits(Set shallowCommits) throws IOException; + /** * Close any resources held by this database. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index 3f167ccce..2aecf63ad 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -16,12 +16,14 @@ import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import org.eclipse.jgit.errors.PackProtocolException; @@ -32,6 +34,7 @@ import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ProgressMonitor; @@ -76,7 +79,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection /** * Maximum number of 'have' lines to send before giving up. *

- * During {@link #negotiate(ProgressMonitor)} we send at most this many + * During {@link #negotiate(ProgressMonitor, boolean, Set)} we send at most this many * commits to the remote peer as 'have' lines without an ACK response before * we give up. */ @@ -210,6 +213,12 @@ public abstract class BasePackFetchConnection extends BasePackConnection private int maxHaves; + private Integer depth; + + private Instant deepenSince; + + private List deepenNots; + /** * RPC state, if {@link BasePackConnection#statelessRPC} is true or protocol * V2 is used. @@ -246,6 +255,9 @@ public BasePackFetchConnection(PackTransport packTransport) { includeTags = transport.getTagOpt() != TagOpt.NO_TAGS; thinPack = transport.isFetchThin(); filterSpec = transport.getFilterSpec(); + depth = transport.getDepth(); + deepenSince = transport.getDeepenSince(); + deepenNots = transport.getDeepenNots(); if (local != null) { walk = new RevWalk(local); @@ -385,9 +397,17 @@ protected void doFetch(final ProgressMonitor monitor, } PacketLineOut output = statelessRPC ? pckState : pckOut; if (sendWants(want, output)) { + boolean mayHaveShallow = depth != null || deepenSince != null || !deepenNots.isEmpty(); + Set shallowCommits = local.getObjectDatabase().getShallowCommits(); + if (isCapableOf(GitProtocolConstants.CAPABILITY_SHALLOW)) { + sendShallow(shallowCommits, output); + } else if (mayHaveShallow) { + throw new PackProtocolException(JGitText.get().shallowNotSupported); + } output.end(); outNeedsEnd = false; - negotiate(monitor); + + negotiate(monitor, mayHaveShallow, shallowCommits); clearState(); @@ -424,10 +444,18 @@ private void doFetchV2(ProgressMonitor monitor, Collection want, for (String capability : getCapabilitiesV2(capabilities)) { pckState.writeString(capability); } + if (!sendWants(want, pckState)) { // We already have everything we wanted. return; } + + Set shallowCommits = local.getObjectDatabase().getShallowCommits(); + if (capabilities.contains(GitProtocolConstants.CAPABILITY_SHALLOW)) { + sendShallow(shallowCommits, pckState); + } else if (depth != null || deepenSince != null || !deepenNots.isEmpty()) { + throw new PackProtocolException(JGitText.get().shallowNotSupported); + } // If we send something, we always close it properly ourselves. outNeedsEnd = false; @@ -458,7 +486,17 @@ private void doFetchV2(ProgressMonitor monitor, Collection want, if (sentDone && line.startsWith("ERR ")) { //$NON-NLS-1$ throw new RemoteRepositoryException(uri, line.substring(4)); } - // "shallow-info", "wanted-refs", and "packfile-uris" would have to be + + if (GitProtocolConstants.SECTION_SHALLOW_INFO.equals(line)) { + line = handleShallowUnshallow(shallowCommits, pckIn); + if (!PacketLineIn.isDelimiter(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0001", line)); //$NON-NLS-1$ + } + line = pckIn.readString(); + } + + // "wanted-refs" and "packfile-uris" would have to be // handled here in that order. if (!GitProtocolConstants.SECTION_PACKFILE.equals(line)) { throw new PackProtocolException( @@ -672,16 +710,19 @@ private boolean sendWants(Collection want, PacketLineOut p) if (objectId == null) { continue; } - try { - if (walk.parseAny(objectId).has(REACHABLE)) { - // We already have this object. Asking for it is - // not a very good idea. - // - continue; + // if depth is set we need to fetch the objects even if they are already available + if (transport.getDepth() == null) { + try { + if (walk.parseAny(objectId).has(REACHABLE)) { + // We already have this object. Asking for it is + // not a very good idea. + // + continue; + } + } catch (IOException err) { + // Its OK, we don't have it, but we want to fix that + // by fetching the object from the other side. } - } catch (IOException err) { - // Its OK, we don't have it, but we want to fix that - // by fetching the object from the other side. } final StringBuilder line = new StringBuilder(46); @@ -773,8 +814,8 @@ else if (wantCapability(line, OPTION_SIDE_BAND)) return line.toString(); } - private void negotiate(ProgressMonitor monitor) throws IOException, - CancelledException { + private void negotiate(ProgressMonitor monitor, boolean mayHaveShallow, Set shallowCommits) + throws IOException, CancelledException { final MutableObjectId ackId = new MutableObjectId(); int resultsPending = 0; int havesSent = 0; @@ -911,6 +952,14 @@ private void negotiate(ProgressMonitor monitor) throws IOException, resultsPending++; } + if (mayHaveShallow) { + String line = handleShallowUnshallow(shallowCommits, pckIn); + if (!PacketLineIn.isEnd(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0000", line)); //$NON-NLS-1$ + } + } + READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) { final AckNackResult anr = pckIn.readACK(ackId); resultsPending--; @@ -1025,6 +1074,50 @@ private void receivePack(final ProgressMonitor monitor, } } + private void sendShallow(Set shallowCommits, PacketLineOut output) throws IOException { + for (ObjectId shallowCommit : shallowCommits) { + output.writeString("shallow " + shallowCommit.name()); //$NON-NLS-1$ + } + + if (depth != null) { + output.writeString("deepen " + depth); //$NON-NLS-1$ + } + + if (deepenSince != null) { + output.writeString("deepen-since " + deepenSince.getEpochSecond()); //$NON-NLS-1$ + } + + if (deepenNots != null) { + for (String deepenNotRef : deepenNots) { + output.writeString("deepen-not " + deepenNotRef); //$NON-NLS-1$ + } + } + } + + private String handleShallowUnshallow(Set advertisedShallowCommits, PacketLineIn input) + throws IOException { + String line = input.readString(); + ObjectDatabase objectDatabase = local.getObjectDatabase(); + HashSet newShallowCommits = new HashSet<>(advertisedShallowCommits); + while (!PacketLineIn.isDelimiter(line) && !PacketLineIn.isEnd(line)) { + if (line.startsWith("shallow ")) { //$NON-NLS-1$ + newShallowCommits.add(ObjectId + .fromString(line.substring("shallow ".length()))); //$NON-NLS-1$ + } else if (line.startsWith("unshallow ")) { //$NON-NLS-1$ + ObjectId unshallow = ObjectId + .fromString(line.substring("unshallow ".length())); //$NON-NLS-1$ + if (!advertisedShallowCommits.contains(unshallow)) { + throw new PackProtocolException(MessageFormat.format(JGitText.get() + .notShallowedUnshallow, unshallow.name())); + } + newShallowCommits.remove(unshallow); + } + line = input.readString(); + } + objectDatabase.setShallowCommits(newShallowCommits); + return line; + } + /** * Notification event delivered just before the pack is received from the * network. This event can be used by RPC such as {@link org.eclipse.jgit.transport.TransportHttp} to diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java index 7bface49d..28c3b6a0f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, 2020 Shawn O. Pearce and others + * Copyright (C) 2008, 2022 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -393,7 +393,7 @@ private boolean askForIsComplete() throws TransportException { ow.markUninteresting(ow.parseAny(ref.getObjectId())); ow.checkConnectivity(); } - return true; + return transport.getDepth() == null; // if depth is set we need to request objects that are already available } catch (MissingObjectException e) { return false; } catch (IOException e) { @@ -516,8 +516,10 @@ private void want(Ref src, RefSpec spec) } if (spec.getDestination() != null) { final TrackingRefUpdate tru = createUpdate(spec, newId); - if (newId.equals(tru.getOldObjectId())) + // if depth is set we need to update the ref + if (newId.equals(tru.getOldObjectId()) && transport.getDepth() == null) { return; + } localUpdates.add(tru); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchRequest.java index 9ebc722ff..0663c5141 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchRequest.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchRequest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -35,7 +35,7 @@ abstract class FetchRequest { final int deepenSince; - final List deepenNotRefs; + final List deepenNots; @Nullable final String agent; @@ -53,7 +53,7 @@ abstract class FetchRequest { * the filter spec * @param clientCapabilities * capabilities sent in the request - * @param deepenNotRefs + * @param deepenNots * Requests that the shallow clone/fetch should be cut at these * specific revisions instead of a depth. * @param deepenSince @@ -66,14 +66,14 @@ abstract class FetchRequest { @NonNull Set clientShallowCommits, @NonNull FilterSpec filterSpec, @NonNull Set clientCapabilities, int deepenSince, - @NonNull List deepenNotRefs, @Nullable String agent) { + @NonNull List deepenNots, @Nullable String agent) { this.wantIds = requireNonNull(wantIds); this.depth = depth; this.clientShallowCommits = requireNonNull(clientShallowCommits); this.filterSpec = requireNonNull(filterSpec); this.clientCapabilities = requireNonNull(clientCapabilities); this.deepenSince = deepenSince; - this.deepenNotRefs = requireNonNull(deepenNotRefs); + this.deepenNots = requireNonNull(deepenNots); this.agent = agent; } @@ -148,8 +148,8 @@ int getDeepenSince() { * @return refs received in "deepen-not" lines. */ @NonNull - List getDeepenNotRefs() { - return deepenNotRefs; + List getDeepenNots() { + return deepenNots; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV0Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV0Request.java index 91adb5e6a..4decb7951 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV0Request.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV0Request.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -11,9 +11,10 @@ import static java.util.Objects.requireNonNull; +import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.eclipse.jgit.annotations.NonNull; @@ -28,15 +29,20 @@ final class FetchV0Request extends FetchRequest { FetchV0Request(@NonNull Set wantIds, int depth, @NonNull Set clientShallowCommits, @NonNull FilterSpec filterSpec, - @NonNull Set clientCapabilities, @Nullable String agent) { + @NonNull Set clientCapabilities, int deepenSince, + @NonNull List deepenNotRefs, @Nullable String agent) { super(wantIds, depth, clientShallowCommits, filterSpec, - clientCapabilities, 0, Collections.emptyList(), agent); + clientCapabilities, deepenSince, deepenNotRefs, agent); } static final class Builder { int depth; + final List deepenNots = new ArrayList<>(); + + int deepenSince; + final Set wantIds = new HashSet<>(); final Set clientShallowCommits = new HashSet<>(); @@ -67,6 +73,50 @@ Builder setDepth(int d) { return this; } + /** + * @return depth set in the request (via a "deepen" line). Defaulting to + * 0 if not set. + */ + int getDepth() { + return depth; + } + + /** + * @return true if there has been at least one "deepen not" line in the + * request so far + */ + boolean hasDeepenNots() { + return !deepenNots.isEmpty(); + } + + /** + * @param deepenNot + * reference received in a "deepen not" line + * @return this builder + */ + Builder addDeepenNot(String deepenNot) { + deepenNots.add(deepenNot); + return this; + } + + /** + * @param value + * Unix timestamp received in a "deepen since" line + * @return this builder + */ + Builder setDeepenSince(int value) { + deepenSince = value; + return this; + } + + /** + * @return shallow since value, sent before in a "deepen since" line. 0 + * by default. + */ + int getDeepenSince() { + return deepenSince; + } + /** * @param shallowOid * object id received in a "shallow" line @@ -110,7 +160,7 @@ Builder setFilterSpec(@NonNull FilterSpec filter) { FetchV0Request build() { return new FetchV0Request(wantIds, depth, clientShallowCommits, - filterSpec, clientCaps, agent); + filterSpec, clientCaps, deepenSince, deepenNots, agent); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java index 50fb9d226..446a3bcc7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -50,7 +50,7 @@ public final class FetchV2Request extends FetchRequest { @NonNull List wantedRefs, @NonNull Set wantIds, @NonNull Set clientShallowCommits, int deepenSince, - @NonNull List deepenNotRefs, int depth, + @NonNull List deepenNots, int depth, @NonNull FilterSpec filterSpec, boolean doneReceived, boolean waitForDone, @NonNull Set clientCapabilities, @@ -58,7 +58,7 @@ public final class FetchV2Request extends FetchRequest { boolean sidebandAll, @NonNull List packfileUriProtocols) { super(wantIds, depth, clientShallowCommits, filterSpec, clientCapabilities, deepenSince, - deepenNotRefs, agent); + deepenNots, agent); this.peerHas = requireNonNull(peerHas); this.wantedRefs = requireNonNull(wantedRefs); this.doneReceived = doneReceived; @@ -140,7 +140,7 @@ static final class Builder { final Set clientShallowCommits = new HashSet<>(); - final List deepenNotRefs = new ArrayList<>(); + final List deepenNots = new ArrayList<>(); final Set clientCapabilities = new HashSet<>(); @@ -240,17 +240,17 @@ int getDepth() { * @return true if there has been at least one "deepen not" line in the * request so far */ - boolean hasDeepenNotRefs() { - return !deepenNotRefs.isEmpty(); + boolean hasDeepenNots() { + return !deepenNots.isEmpty(); } /** - * @param deepenNotRef + * @param deepenNot * reference received in a "deepen not" line * @return this builder */ - Builder addDeepenNotRef(String deepenNotRef) { - deepenNotRefs.add(deepenNotRef); + Builder addDeepenNot(String deepenNot) { + deepenNots.add(deepenNot); return this; } @@ -350,7 +350,7 @@ Builder addPackfileUriProtocol(@NonNull String value) { */ FetchV2Request build() { return new FetchV2Request(peerHas, wantedRefs, wantIds, - clientShallowCommits, deepenSince, deepenNotRefs, + clientShallowCommits, deepenSince, deepenNots, depth, filterSpec, doneReceived, waitForDone, clientCapabilities, agent, Collections.unmodifiableList(serverOptions), sidebandAll, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java index aaa9308ac..24ea552ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2008, 2013 Google Inc. * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, 2020 Shawn O. Pearce and others + * Copyright (C) 2008, 2022 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -233,6 +233,13 @@ public final class GitProtocolConstants { */ public static final String CAPABILITY_SERVER_OPTION = "server-option"; //$NON-NLS-1$ + /** + * The server supports the receiving of shallow options. + * + * @since 6.3 + */ + public static final String CAPABILITY_SHALLOW = "shallow"; //$NON-NLS-1$ + /** * Option for passing application-specific options to the server. * @@ -307,6 +314,13 @@ public final class GitProtocolConstants { */ public static final String SECTION_PACKFILE = "packfile"; //$NON-NLS-1$ + /** + * Protocol V2 shallow-info section header. + * + * @since 6.3 + */ + public static final String SECTION_SHALLOW_INFO = "shallow-info"; //$NON-NLS-1$ + /** * Protocol announcement for protocol version 1. This is the same as V0, * except for this initial line. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV0Parser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV0Parser.java index f8c51c180..4ddcb9941 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV0Parser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV0Parser.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -77,10 +77,42 @@ FetchV0Request recvWants(PacketLineIn pckIn) MessageFormat.format(JGitText.get().invalidDepth, Integer.valueOf(depth))); } + if (reqBuilder.getDeepenSince() != 0) { + throw new PackProtocolException( + JGitText.get().deepenSinceWithDeepen); + } + if (reqBuilder.hasDeepenNots()) { + throw new PackProtocolException( + JGitText.get().deepenNotWithDeepen); + } reqBuilder.setDepth(depth); continue; } + if (line.startsWith("deepen-not ")) { //$NON-NLS-1$ + reqBuilder.addDeepenNot(line.substring(11)); + if (reqBuilder.getDepth() != 0) { + throw new PackProtocolException( + JGitText.get().deepenNotWithDeepen); + } + continue; + } + + if (line.startsWith("deepen-since ")) { //$NON-NLS-1$ + // TODO: timestamps should be long + int ts = Integer.parseInt(line.substring(13)); + if (ts <= 0) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().invalidTimestamp, line)); + } + if (reqBuilder.getDepth() != 0) { + throw new PackProtocolException( + JGitText.get().deepenSinceWithDeepen); + } + reqBuilder.setDeepenSince(ts); + continue; + } + if (line.startsWith("shallow ")) { //$NON-NLS-1$ reqBuilder.addClientShallowCommit( ObjectId.fromString(line.substring(8))); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java index 6cec4b9a3..e502831a2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Google LLC. and others + * Copyright (C) 2018, 2022 Google LLC. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -149,13 +149,13 @@ FetchV2Request parseFetchRequest(PacketLineIn pckIn) throw new PackProtocolException( JGitText.get().deepenSinceWithDeepen); } - if (reqBuilder.hasDeepenNotRefs()) { + if (reqBuilder.hasDeepenNots()) { throw new PackProtocolException( JGitText.get().deepenNotWithDeepen); } reqBuilder.setDepth(parsedDepth); } else if (line2.startsWith("deepen-not ")) { //$NON-NLS-1$ - reqBuilder.addDeepenNotRef(line2.substring(11)); + reqBuilder.addDeepenNot(line2.substring(11)); if (reqBuilder.getDepth() != 0) { throw new PackProtocolException( JGitText.get().deepenNotWithDeepen); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index 3222d6330..7cea99847 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -27,6 +27,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -784,6 +785,12 @@ static String findTrackingRefName(final String remoteName, private PrePushHook prePush; + private Integer depth; + + private Instant deepenSince; + + private List deepenNots = new ArrayList<>(); + @Nullable TransferConfig.ProtocolVersion protocol; @@ -1086,6 +1093,83 @@ public final void setFilterSpec(@NonNull FilterSpec filter) { filterSpec = requireNonNull(filter); } + + /** + * Retrieves the depth for a shallow clone. + * + * @return the depth, or {@code null} if none set + * @since 6.3 + */ + public final Integer getDepth() { + return depth; + } + + /** + * Limits fetching to the specified number of commits from the tip of each + * remote branch history. + * + * @param depth + * the depth + * @since 6.3 + */ + public final void setDepth(int depth) { + if (depth < 1) { + throw new IllegalArgumentException(JGitText.get().depthMustBeAt1); + } + this.depth = Integer.valueOf(depth); + } + + /** + * Limits fetching to the specified number of commits from the tip of each + * remote branch history. + * + * @param depth + * the depth, or {@code null} to unset the depth + * @since 6.3 + */ + public final void setDepth(Integer depth) { + if (depth != null && depth.intValue() < 1) { + throw new IllegalArgumentException(JGitText.get().depthMustBeAt1); + } + this.depth = depth; + } + + /** + * @return the deepen-since for a shallow clone + * @since 6.3 + */ + public final Instant getDeepenSince() { + return deepenSince; + } + + /** + * Deepen or shorten the history of a shallow repository to include all reachable commits after a specified time. + * + * @param deepenSince the deepen-since. Must not be {@code null} + * @since 6.3 + */ + public final void setDeepenSince(@NonNull Instant deepenSince) { + this.deepenSince = deepenSince; + } + + /** + * @return the deepen-not for a shallow clone + * @since 6.3 + */ + public final List getDeepenNots() { + return deepenNots; + } + + /** + * Deepen or shorten the history of a shallow repository to exclude commits reachable from a specified remote branch or tag. + * + * @param deepenNots the deepen-not. Must not be {@code null} + * @since 6.3 + */ + public final void setDeepenNots(@NonNull List deepenNots) { + this.deepenNots = deepenNots; + } + /** * Apply provided remote configuration on this transport. * @@ -1230,7 +1314,7 @@ public void setPushOptions(List pushOptions) { * @param toFetch * specification of refs to fetch locally. May be null or the * empty collection to use the specifications from the - * RemoteConfig. May contains regular and negative + * RemoteConfig. May contains regular and negative * {@link RefSpec}s. Source for each regular RefSpec can't * be null. * @return information describing the tracking refs updated. @@ -1266,7 +1350,7 @@ public FetchResult fetch(final ProgressMonitor monitor, * @param toFetch * specification of refs to fetch locally. May be null or the * empty collection to use the specifications from the - * RemoteConfig. May contains regular and negative + * RemoteConfig. May contain regular and negative * {@link RefSpec}s. Source for each regular RefSpec can't * be null. * @param branch diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index dcd806a3d..409161d58 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008, 2020 Google Inc. and others + * Copyright (C) 2008, 2022 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -10,6 +10,7 @@ package org.eclipse.jgit.transport; +import static java.util.Collections.emptyList; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; import static org.eclipse.jgit.lib.Constants.R_TAGS; @@ -1033,6 +1034,7 @@ private void service(PacketLineOut pckOut) throws IOException { // writing a response. Buffer the response until then. PackStatistics.Accumulator accumulator = new PackStatistics.Accumulator(); List unshallowCommits = new ArrayList<>(); + List deepenNots = emptyList(); FetchRequest req; try { if (biDirectionalPipe) @@ -1071,13 +1073,14 @@ else if (requestValidator instanceof AnyRequestValidator) verifyClientShallow(req.getClientShallowCommits()); } - if (req.getDepth() != 0 || req.getDeepenSince() != 0) { + deepenNots = parseDeepenNots(req.getDeepenNots()); + if (req.getDepth() != 0 || req.getDeepenSince() != 0 || !req.getDeepenNots().isEmpty()) { computeShallowsAndUnshallows(req, shallow -> { pckOut.writeString("shallow " + shallow.name() + '\n'); //$NON-NLS-1$ }, unshallow -> { pckOut.writeString("unshallow " + unshallow.name() + '\n'); //$NON-NLS-1$ unshallowCommits.add(unshallow); - }, Collections.emptyList()); + }, deepenNots); pckOut.end(); } @@ -1109,7 +1112,7 @@ else if (requestValidator instanceof AnyRequestValidator) if (sendPack) { sendPack(accumulator, req, refs == null ? null : refs.values(), - unshallowCommits, Collections.emptyList(), pckOut); + unshallowCommits, deepenNots, pckOut); } } @@ -1188,15 +1191,7 @@ private void fetchV2(PacketLineOut pckOut) throws IOException { // TODO(ifrade): Refactor to pass around the Request object, instead of // copying data back to class fields - List deepenNots = new ArrayList<>(); - for (String s : req.getDeepenNotRefs()) { - Ref ref = findRef(s); - if (ref == null) { - throw new PackProtocolException(MessageFormat - .format(JGitText.get().invalidRefName, s)); - } - deepenNots.add(ref.getObjectId()); - } + List deepenNots = parseDeepenNots(req.getDeepenNots()); Map wantedRefs = wantedRefs(req); // TODO(ifrade): Avoid mutating the parsed request. @@ -1206,7 +1201,7 @@ private void fetchV2(PacketLineOut pckOut) throws IOException { boolean sectionSent = false; boolean mayHaveShallow = req.getDepth() != 0 || req.getDeepenSince() != 0 - || !req.getDeepenNotRefs().isEmpty(); + || !req.getDeepenNots().isEmpty(); List shallowCommits = new ArrayList<>(); List unshallowCommits = new ArrayList<>(); @@ -2476,6 +2471,24 @@ private void addTagChain( } } + private List parseDeepenNots(List deepenNots) + throws IOException { + List result = new ArrayList<>(); + for (String s : deepenNots) { + if (ObjectId.isId(s)) { + result.add(ObjectId.fromString(s)); + } else { + Ref ref = findRef(s); + if (ref == null) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().invalidRefName, s)); + } + result.add(ref.getObjectId()); + } + } + return result; + } + private static class ResponseBufferedOutputStream extends OutputStream { private final OutputStream rawOut;