diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/commit-graph.v1 b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/commit-graph.v1 new file mode 100644 index 000000000..941c0a7ce Binary files /dev/null and b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/commit-graph.v1 differ diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphBuilderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphBuilderTest.java new file mode 100644 index 000000000..8ecf5df4e --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphBuilderTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.junit.Test; + +public class CommitGraphBuilderTest { + + @Test + public void testRepeatedChunk() throws Exception { + byte[] buffer = new byte[2048]; + + CommitGraphBuilder builder1 = CommitGraphBuilder.builder(); + builder1.addOidFanout(buffer); + Exception e1 = assertThrows(CommitGraphFormatException.class, () -> { + builder1.addOidFanout(buffer); + }); + assertEquals("commit-graph chunk id 0x4f494446 appears multiple times", + e1.getMessage()); + + CommitGraphBuilder builder2 = CommitGraphBuilder.builder(); + builder2.addOidLookUp(buffer); + Exception e2 = assertThrows(CommitGraphFormatException.class, () -> { + builder2.addOidLookUp(buffer); + }); + assertEquals("commit-graph chunk id 0x4f49444c appears multiple times", + e2.getMessage()); + + CommitGraphBuilder builder3 = CommitGraphBuilder.builder(); + builder3.addCommitData(buffer); + Exception e3 = assertThrows(CommitGraphFormatException.class, () -> { + builder3.addCommitData(buffer); + }); + assertEquals("commit-graph chunk id 0x43444154 appears multiple times", + e3.getMessage()); + + CommitGraphBuilder builder4 = CommitGraphBuilder.builder(); + builder4.addExtraList(buffer); + Exception e4 = assertThrows(CommitGraphFormatException.class, () -> { + builder4.addExtraList(buffer); + }); + assertEquals("commit-graph chunk id 0x45444745 appears multiple times", + e4.getMessage()); + } + + @Test + public void testNeededChunk() { + byte[] buffer = new byte[2048]; + + Exception e1 = assertThrows(CommitGraphFormatException.class, () -> { + CommitGraphBuilder.builder().addOidLookUp(buffer) + .addCommitData(buffer).build(); + }); + assertEquals("commit-graph 0x4f494446 chunk has not been loaded", + e1.getMessage()); + + Exception e2 = assertThrows(CommitGraphFormatException.class, () -> { + CommitGraphBuilder.builder().addOidFanout(buffer) + .addCommitData(buffer).build(); + }); + assertEquals("commit-graph 0x4f49444c chunk has not been loaded", + e2.getMessage()); + + Exception e3 = assertThrows(CommitGraphFormatException.class, () -> { + CommitGraphBuilder.builder().addOidFanout(buffer) + .addOidLookUp(buffer).build(); + }); + assertEquals("commit-graph 0x43444154 chunk has not been loaded", + e3.getMessage()); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphLoaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphLoaderTest.java new file mode 100644 index 000000000..fd427a117 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphLoaderTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph.CommitData; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.lib.ObjectId; +import org.junit.Test; + +/** + * Test CommitGraphLoader by reading the commit-graph file generated by Cgit. + */ +public class CommitGraphLoaderTest { + + private CommitGraph commitGraph; + + @Test + public void readCommitGraphV1() throws Exception { + commitGraph = CommitGraphLoader + .open(JGitTestUtil.getTestResourceFile("commit-graph.v1")); + assertNotNull(commitGraph); + assertEquals(10, commitGraph.getCommitCnt()); + verifyGraphObjectIndex(); + + assertCommitData("85b0176af27fa1640868f061f224d01e0b295f59", + new int[] { 5, 6 }, 1670570408L, 3, 0); + assertCommitData("d4f7c00aab3f0160168c9e5991abb6194a4e0d9e", + new int[] {}, 1670569901L, 1, 1); + assertCommitData("4d03aaf9c20c97d6ccdc05cb7f146b1deb6c01d5", + new int[] { 5 }, 1670570119L, 3, 2); + assertCommitData("a2f409b753880bf83b18bfb433dd340a6185e8be", + new int[] { 7 }, 1670569935L, 3, 3); + assertCommitData("431343847343979bbe31127ed905a24fed9a636c", + new int[] { 3, 2, 8 }, 1670570644L, 4, 4); + assertCommitData("c3f745ad8928ef56b5dbf33740fc8ede6b598290", + new int[] { 1 }, 1670570106L, 2, 5); + assertCommitData("95b12422c8ea4371e54cd58925eeed9d960ff1f0", + new int[] { 1 }, 1670570163L, 2, 6); + assertCommitData("de0ea882503cdd9c984c0a43238014569a123cac", + new int[] { 1 }, 1670569921L, 2, 7); + assertCommitData("102c9d6481559b1a113eb66bf55085903de6fb00", + new int[] { 6 }, 1670570616L, 3, 8); + assertCommitData("b5de2a84867f8ffc6321649dabf8c0680661ec03", + new int[] { 7, 5 }, 1670570364L, 3, 9); + } + + private void verifyGraphObjectIndex() { + for (int i = 0; i < commitGraph.getCommitCnt(); i++) { + ObjectId id = commitGraph.getObjectId(i); + int pos = commitGraph.findGraphPosition(id); + assertEquals(i, pos); + } + } + + private void assertCommitData(String expectedTree, int[] expectedParents, + long expectedCommitTime, int expectedGeneration, int graphPos) { + CommitData commitData = commitGraph.getCommitData(graphPos); + assertEquals(ObjectId.fromString(expectedTree), commitData.getTree()); + assertArrayEquals(expectedParents, commitData.getParents()); + assertEquals(expectedCommitTime, commitData.getCommitTime()); + assertEquals(expectedGeneration, commitData.getGeneration()); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java new file mode 100644 index 000000000..97976564d --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphTest.java @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import static org.eclipse.jgit.lib.Constants.COMMIT_GENERATION_UNKNOWN; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.junit.Before; +import org.junit.Test; + +/** + * Test writing and then reading the commit-graph. + */ +public class CommitGraphTest extends RepositoryTestCase { + + private TestRepository tr; + + private CommitGraph commitGraph; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + tr = new TestRepository<>(db, new RevWalk(db), mockSystemReader); + } + + @Test + public void testGraphWithSingleCommit() throws Exception { + RevCommit root = commit(); + writeAndReadCommitGraph(Collections.singleton(root)); + verifyCommitGraph(); + assertEquals(1, getGenerationNumber(root)); + } + + @Test + public void testGraphWithManyParents() throws Exception { + int parentsNum = 40; + RevCommit root = commit(); + + RevCommit[] parents = new RevCommit[parentsNum]; + for (int i = 0; i < parents.length; i++) { + parents[i] = commit(root); + } + RevCommit tip = commit(parents); + + Set wants = Collections.singleton(tip); + writeAndReadCommitGraph(wants); + assertEquals(parentsNum + 2, commitGraph.getCommitCnt()); + verifyCommitGraph(); + + assertEquals(1, getGenerationNumber(root)); + for (RevCommit parent : parents) { + assertEquals(2, getGenerationNumber(parent)); + } + assertEquals(3, getGenerationNumber(tip)); + } + + @Test + public void testGraphLinearHistory() throws Exception { + int commitNum = 20; + RevCommit[] commits = new RevCommit[commitNum]; + for (int i = 0; i < commitNum; i++) { + if (i == 0) { + commits[i] = commit(); + } else { + commits[i] = commit(commits[i - 1]); + } + } + + Set wants = Collections.singleton(commits[commitNum - 1]); + writeAndReadCommitGraph(wants); + assertEquals(commitNum, commitGraph.getCommitCnt()); + verifyCommitGraph(); + for (int i = 0; i < commitNum; i++) { + assertEquals(i + 1, getGenerationNumber(commits[i])); + } + } + + @Test + public void testGraphWithMerges() throws Exception { + RevCommit c1 = commit(); + RevCommit c2 = commit(c1); + RevCommit c3 = commit(c2); + RevCommit c4 = commit(c1); + RevCommit c5 = commit(c4); + RevCommit c6 = commit(c1); + RevCommit c7 = commit(c6); + + RevCommit m1 = commit(c2, c4); + RevCommit m2 = commit(c4, c6); + RevCommit m3 = commit(c3, c5, c7); + + Set wants = new HashSet<>(); + + /* + *
+		 * current graph structure:
+		 *    M1
+		 *   /  \
+		 *  2    4
+		 *  |___/
+		 *  1
+		 * 
+ */ + wants.add(m1); + writeAndReadCommitGraph(wants); + assertEquals(4, commitGraph.getCommitCnt()); + verifyCommitGraph(); + + /* + *
+		 * current graph structure:
+		 *    M1   M2
+		 *   /  \ /  \
+		 *  2    4    6
+		 *  |___/____/
+		 *  1
+		 * 
+ */ + wants.add(m2); + writeAndReadCommitGraph(wants); + assertEquals(6, commitGraph.getCommitCnt()); + verifyCommitGraph(); + + /* + *
+		 * current graph structure:
+		 *
+		 *    __M3___
+		 *   /   |   \
+		 *  3 M1 5 M2 7
+		 *  |/  \|/  \|
+		 *  2    4    6
+		 *  |___/____/
+		 *  1
+		 * 
+ */ + wants.add(m3); + writeAndReadCommitGraph(wants); + assertEquals(10, commitGraph.getCommitCnt()); + verifyCommitGraph(); + + /* + *
+		 * current graph structure:
+		 *       8
+		 *       |
+		 *    __M3___
+		 *   /   |   \
+		 *  3 M1 5 M2 7
+		 *  |/  \|/  \|
+		 *  2    4    6
+		 *  |___/____/
+		 *  1
+		 * 
+ */ + RevCommit c8 = commit(m3); + wants.add(c8); + writeAndReadCommitGraph(wants); + assertEquals(11, commitGraph.getCommitCnt()); + verifyCommitGraph(); + + assertEquals(getGenerationNumber(c1), 1); + assertEquals(getGenerationNumber(c2), 2); + assertEquals(getGenerationNumber(c4), 2); + assertEquals(getGenerationNumber(c6), 2); + assertEquals(getGenerationNumber(c3), 3); + assertEquals(getGenerationNumber(c5), 3); + assertEquals(getGenerationNumber(c7), 3); + assertEquals(getGenerationNumber(m1), 3); + assertEquals(getGenerationNumber(m2), 3); + assertEquals(getGenerationNumber(m3), 4); + assertEquals(getGenerationNumber(c8), 5); + } + + void writeAndReadCommitGraph(Set wants) throws Exception { + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + try (RevWalk walk = new RevWalk(db)) { + CommitGraphWriter writer = new CommitGraphWriter( + GraphCommits.fromWalk(m, wants, walk)); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + writer.write(m, os); + InputStream inputStream = new ByteArrayInputStream( + os.toByteArray()); + commitGraph = CommitGraphLoader.read(inputStream); + } + } + + void verifyCommitGraph() throws Exception { + try (RevWalk walk = new RevWalk(db)) { + for (int i = 0; i < commitGraph.getCommitCnt(); i++) { + ObjectId objId = commitGraph.getObjectId(i); + + // check the objectId index of commit-graph + int pos = commitGraph.findGraphPosition(objId); + assertEquals(i, pos); + + // check the commit meta of commit-graph + CommitGraph.CommitData commit = commitGraph.getCommitData(i); + int[] pList = commit.getParents(); + + RevCommit expect = walk.lookupCommit(objId); + walk.parseBody(expect); + + assertEquals(expect.getCommitTime(), commit.getCommitTime()); + assertEquals(expect.getTree(), commit.getTree()); + assertEquals(expect.getParentCount(), pList.length); + + if (pList.length > 0) { + ObjectId[] parents = new ObjectId[pList.length]; + for (int j = 0; j < parents.length; j++) { + parents[j] = commitGraph.getObjectId(pList[j]); + } + assertArrayEquals(expect.getParents(), parents); + } + } + } + } + + int getGenerationNumber(ObjectId id) { + int graphPos = commitGraph.findGraphPosition(id); + CommitGraph.CommitData commitData = commitGraph.getCommitData(graphPos); + if (commitData != null) { + return commitData.getGeneration(); + } + return COMMIT_GENERATION_UNKNOWN; + } + + RevCommit commit(RevCommit... parents) throws Exception { + return tr.commit(parents); + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 9b9e6d5a2..9c918ad41 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -143,6 +143,10 @@ collisionOn=Collision on {0} commandClosedStderrButDidntExit=Command {0} closed stderr stream but didn''t exit within timeout {1} seconds commandRejectedByHook=Rejected by "{0}" hook.\n{1} commandWasCalledInTheWrongState=Command {0} was called in the wrong state +commitGraphChunkNeeded=commit-graph 0x{0} chunk has not been loaded +commitGraphChunkRepeated=commit-graph chunk id 0x{0} appears multiple times +commitGraphChunkUnknown=unknown commit-graph chunk: 0x{0} +commitGraphFileIsTooLargeForJgit=commit-graph file is too large for jgit commitGraphWritingCancelled=commit-graph writing was canceled commitMessageNotSpecified=commit message not specified commitOnRepoWithoutHEADCurrentlyNotSupported=Commit on repo without HEAD currently not supported @@ -385,6 +389,7 @@ invalidDepth=Invalid depth: {0} invalidEncoding=Invalid encoding from git config i18n.commitEncoding: {0} invalidEncryption=Invalid encryption invalidExpandWildcard=ExpandFromSource on a refspec that can have mismatched wildcards does not make sense. +invalidExtraEdgeListPosition=Invalid position in Extra Edge List chunk: {0} invalidFilter=Invalid filter: {0} invalidGitdirRef = Invalid .git reference in file ''{0}'' invalidGitModules=Invalid .gitmodules file @@ -515,6 +520,7 @@ noSuchRefKnown=no such ref: {0} noSuchSubmodule=no such submodule {0} notABoolean=Not a boolean: {0} notABundle=not a bundle +notACommitGraph=not a commit-graph notADIRCFile=Not a DIRC file. notAGitDirectory=not a git directory notAPACKFile=Not a PACK file. @@ -797,6 +803,7 @@ unlockLockFileFailed=Unlocking LockFile ''{0}'' failed unmergedPath=Unmerged path: {0} unmergedPaths=Repository contains unmerged paths unpackException=Exception while parsing pack stream +unreadableCommitGraph=Unreadable commit-graph: {0} unreadablePackIndex=Unreadable pack index: {0} unrecognizedPackExtension=Unrecognized pack extension: {0} unrecognizedRef=Unrecognized ref: {0} @@ -804,6 +811,7 @@ unsetMark=Mark not set unsupportedAlternates=Alternates not supported unsupportedArchiveFormat=Unknown archive format ''{0}'' unsupportedCommand0=unsupported command 0 +unsupportedCommitGraphVersion=Unsupported commit-graph version: {0} unsupportedEncryptionAlgorithm=Unsupported encryption algorithm: {0} unsupportedEncryptionVersion=Unsupported encryption version: {0} unsupportedGC=Unsupported garbage collector for repository type: {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 7e741e09e..3300742f8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -171,6 +171,10 @@ public static JGitText get() { /***/ public String commandClosedStderrButDidntExit; /***/ public String commandRejectedByHook; /***/ public String commandWasCalledInTheWrongState; + /***/ public String commitGraphChunkNeeded; + /***/ public String commitGraphChunkRepeated; + /***/ public String commitGraphChunkUnknown; + /***/ public String commitGraphFileIsTooLargeForJgit; /***/ public String commitGraphWritingCancelled; /***/ public String commitMessageNotSpecified; /***/ public String commitOnRepoWithoutHEADCurrentlyNotSupported; @@ -413,6 +417,7 @@ public static JGitText get() { /***/ public String invalidEncoding; /***/ public String invalidEncryption; /***/ public String invalidExpandWildcard; + /***/ public String invalidExtraEdgeListPosition; /***/ public String invalidFilter; /***/ public String invalidGitdirRef; /***/ public String invalidGitModules; @@ -543,6 +548,7 @@ public static JGitText get() { /***/ public String noSuchSubmodule; /***/ public String notABoolean; /***/ public String notABundle; + /***/ public String notACommitGraph; /***/ public String notADIRCFile; /***/ public String notAGitDirectory; /***/ public String notAPACKFile; @@ -825,6 +831,7 @@ public static JGitText get() { /***/ public String unmergedPath; /***/ public String unmergedPaths; /***/ public String unpackException; + /***/ public String unreadableCommitGraph; /***/ public String unreadablePackIndex; /***/ public String unrecognizedPackExtension; /***/ public String unrecognizedRef; @@ -832,6 +839,7 @@ public static JGitText get() { /***/ public String unsupportedAlternates; /***/ public String unsupportedArchiveFormat; /***/ public String unsupportedCommand0; + /***/ public String unsupportedCommitGraphVersion; /***/ public String unsupportedEncryptionAlgorithm; /***/ public String unsupportedEncryptionVersion; /***/ public String unsupportedGC; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraph.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraph.java new file mode 100644 index 000000000..162e0e2cb --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraph.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; + +/** + * The CommitGraph is a supplemental data structure that accelerates commit + * graph walks. + *

+ * If a user downgrades or disables the core.commitGraph config + * setting, then the existing object database is sufficient. + *

+ *

+ * It stores the commit graph structure along with some extra metadata to speed + * up graph walks. By listing commit OIDs in lexicographic order, we can + * identify an integer position for each commit and refer to the parents of a + * commit using those integer positions. We use binary search to find initial + * commits and then use the integer positions for fast lookups during the walk. + *

+ */ +public interface CommitGraph { + + /** + * Find the position in the commit-graph of the commit. + *

+ * The position can only be used within the CommitGraph Instance you got it + * from. That's because the graph position of the same commit may be + * different in CommitGraph obtained at different times (eg., regenerated + * new commit-graph). + * + * @param commit + * the commit for which the commit-graph position will be found. + * @return the commit-graph position or -1 if the object was not found. + */ + int findGraphPosition(AnyObjectId commit); + + /** + * Get the metadata of a commit。 + *

+ * This function runs in time O(1). + *

+ * In the process of commit history traversal, + * {@link CommitData#getParents()} makes us get the graphPos of the commit's + * parents in advance, so that we can avoid O(logN) lookup and use O(1) + * lookup instead. + * + * @param graphPos + * the position in the commit-graph of the object. + * @return the metadata of a commit or null if it's not found. + */ + CommitData getCommitData(int graphPos); + + /** + * Get the object at the commit-graph position. + * + * @param graphPos + * the position in the commit-graph of the object. + * @return the ObjectId or null if it's not found. + */ + ObjectId getObjectId(int graphPos); + + /** + * Obtain the total number of commits described by this commit-graph. + * + * @return number of commits in this commit-graph. + */ + long getCommitCnt(); + + /** + * Metadata of a commit in commit data chunk. + */ + interface CommitData { + + /** + * Get a reference to this commit's tree. + * + * @return tree of this commit. + */ + ObjectId getTree(); + + /** + * Obtain an array of all parents. + *

+ * The method only provides the graph positions of parents in + * commit-graph, call {@link CommitGraph#getObjectId(int)} to get the + * real objectId. + * + * @return the array of parents. + */ + int[] getParents(); + + /** + * Time from the "committer" line. + * + * @return the commit time in seconds since EPOCH. + */ + long getCommitTime(); + + /** + * Get the generation number (the distance from the root) of the commit. + * + * @return the generation number or + * {@link org.eclipse.jgit.lib.Constants#COMMIT_GENERATION_NOT_COMPUTED} + * if the writer didn't calculate it. + */ + int getGeneration(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphBuilder.java new file mode 100644 index 000000000..a6af3bc59 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphBuilder.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_COMMIT_DATA; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_FANOUT; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_LOOKUP; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; + +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; + +/** + * Builder for {@link CommitGraph}. + */ +class CommitGraphBuilder { + + private final int hashLength; + + private byte[] oidFanout; + + private byte[] oidLookup; + + private byte[] commitData; + + private byte[] extraList; + + /** @return A builder of {@link CommitGraph}. */ + static CommitGraphBuilder builder() { + return new CommitGraphBuilder(OBJECT_ID_LENGTH); + } + + private CommitGraphBuilder(int hashLength) { + this.hashLength = hashLength; + } + + CommitGraphBuilder addOidFanout(byte[] buffer) + throws CommitGraphFormatException { + assertChunkNotSeenYet(oidFanout, CHUNK_ID_OID_FANOUT); + oidFanout = buffer; + return this; + } + + CommitGraphBuilder addOidLookUp(byte[] buffer) + throws CommitGraphFormatException { + assertChunkNotSeenYet(oidLookup, CHUNK_ID_OID_LOOKUP); + oidLookup = buffer; + return this; + } + + CommitGraphBuilder addCommitData(byte[] buffer) + throws CommitGraphFormatException { + assertChunkNotSeenYet(commitData, CHUNK_ID_COMMIT_DATA); + commitData = buffer; + return this; + } + + CommitGraphBuilder addExtraList(byte[] buffer) + throws CommitGraphFormatException { + assertChunkNotSeenYet(extraList, CHUNK_ID_EXTRA_EDGE_LIST); + extraList = buffer; + return this; + } + + CommitGraph build() throws CommitGraphFormatException { + assertChunkNotNull(oidFanout, CHUNK_ID_OID_FANOUT); + assertChunkNotNull(oidLookup, CHUNK_ID_OID_LOOKUP); + assertChunkNotNull(commitData, CHUNK_ID_COMMIT_DATA); + + GraphObjectIndex index = new GraphObjectIndex(hashLength, oidFanout, + oidLookup); + GraphCommitData commitDataChunk = new GraphCommitData(hashLength, + commitData, extraList); + return new CommitGraphV1(index, commitDataChunk); + } + + private void assertChunkNotNull(Object object, int chunkId) + throws CommitGraphFormatException { + if (object == null) { + throw new CommitGraphFormatException( + MessageFormat.format(JGitText.get().commitGraphChunkNeeded, + Integer.toHexString(chunkId))); + } + } + + private void assertChunkNotSeenYet(Object object, int chunkId) + throws CommitGraphFormatException { + if (object != null) { + throw new CommitGraphFormatException(MessageFormat.format( + JGitText.get().commitGraphChunkRepeated, + Integer.toHexString(chunkId))); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphFormatException.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphFormatException.java new file mode 100644 index 000000000..352bf4b9e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphFormatException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import java.io.IOException; + +/** + * Thrown when a commit-graph file's format is different from we expected + */ +public class CommitGraphFormatException extends IOException { + + private static final long serialVersionUID = 1L; + + /** + * Construct an exception. + * + * @param why + * description of the type of error. + */ + CommitGraphFormatException(String why) { + super(why); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphLoader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphLoader.java new file mode 100644 index 000000000..571f5f4eb --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphLoader.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_COMMIT_DATA; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_EXTRA_EDGE_LIST; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_FANOUT; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_ID_OID_LOOKUP; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.CHUNK_LOOKUP_WIDTH; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_GRAPH_MAGIC; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.io.SilentFileInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The loader returns the representation of the commit-graph file content. + */ +public class CommitGraphLoader { + + private final static Logger LOG = LoggerFactory + .getLogger(CommitGraphLoader.class); + + /** + * Open an existing commit-graph file for reading. + *

+ * The format of the file will be automatically detected and a proper access + * implementation for that format will be constructed and returned to the + * caller. The file may or may not be held open by the returned instance. + * + * @param graphFile + * existing commit-graph to read. + * @return a copy of the commit-graph file in memory + * @throws FileNotFoundException + * the file does not exist. + * @throws CommitGraphFormatException + * commit-graph file's format is different from we expected. + * @throws java.io.IOException + * the file exists but could not be read due to security errors + * or unexpected data corruption. + */ + public static CommitGraph open(File graphFile) throws FileNotFoundException, + CommitGraphFormatException, IOException { + try (SilentFileInputStream fd = new SilentFileInputStream(graphFile)) { + try { + return read(fd); + } catch (CommitGraphFormatException fe) { + throw fe; + } catch (IOException ioe) { + throw new IOException(MessageFormat.format( + JGitText.get().unreadableCommitGraph, + graphFile.getAbsolutePath()), ioe); + } + } + } + + /** + * Read an existing commit-graph file from a buffered stream. + *

+ * The format of the file will be automatically detected and a proper access + * implementation for that format will be constructed and returned to the + * caller. The file may or may not be held open by the returned instance. + * + * @param fd + * stream to read the commit-graph file from. The stream must be + * buffered as some small IOs are performed against the stream. + * The caller is responsible for closing the stream. + * + * @return a copy of the commit-graph file in memory + * @throws CommitGraphFormatException + * the commit-graph file's format is different from we expected. + * @throws java.io.IOException + * the stream cannot be read. + */ + public static CommitGraph read(InputStream fd) + throws CommitGraphFormatException, IOException { + byte[] hdr = new byte[8]; + IO.readFully(fd, hdr, 0, hdr.length); + + int magic = NB.decodeInt32(hdr, 0); + if (magic != COMMIT_GRAPH_MAGIC) { + throw new CommitGraphFormatException( + JGitText.get().notACommitGraph); + } + + // Read the hash version (1 byte) + // 1 => SHA-1 + // 2 => SHA-256 nonsupport now + int hashVersion = hdr[5]; + if (hashVersion != 1) { + throw new CommitGraphFormatException( + JGitText.get().incorrectOBJECT_ID_LENGTH); + } + + // Check commit-graph version + int v = hdr[4]; + if (v != 1) { + throw new CommitGraphFormatException(MessageFormat.format( + JGitText.get().unsupportedCommitGraphVersion, + Integer.valueOf(v))); + } + + // Read the number of "chunkOffsets" (1 byte) + int numberOfChunks = hdr[6]; + + // hdr[7] is the number of base commit-graphs, which is not supported in + // current version + + byte[] lookupBuffer = new byte[CHUNK_LOOKUP_WIDTH + * (numberOfChunks + 1)]; + IO.readFully(fd, lookupBuffer, 0, lookupBuffer.length); + List chunks = new ArrayList<>(numberOfChunks + 1); + for (int i = 0; i <= numberOfChunks; i++) { + // chunks[numberOfChunks] is just a marker, in order to record the + // length of the last chunk. + int id = NB.decodeInt32(lookupBuffer, i * 12); + long offset = NB.decodeInt64(lookupBuffer, i * 12 + 4); + chunks.add(new ChunkSegment(id, offset)); + } + + CommitGraphBuilder builder = CommitGraphBuilder.builder(); + for (int i = 0; i < numberOfChunks; i++) { + long chunkOffset = chunks.get(i).offset; + int chunkId = chunks.get(i).id; + long len = chunks.get(i + 1).offset - chunkOffset; + + if (len > Integer.MAX_VALUE - 8) { // http://stackoverflow.com/a/8381338 + throw new CommitGraphFormatException( + JGitText.get().commitGraphFileIsTooLargeForJgit); + } + + byte buffer[] = new byte[(int) len]; + IO.readFully(fd, buffer, 0, buffer.length); + + switch (chunkId) { + case CHUNK_ID_OID_FANOUT: + builder.addOidFanout(buffer); + break; + case CHUNK_ID_OID_LOOKUP: + builder.addOidLookUp(buffer); + break; + case CHUNK_ID_COMMIT_DATA: + builder.addCommitData(buffer); + break; + case CHUNK_ID_EXTRA_EDGE_LIST: + builder.addExtraList(buffer); + break; + default: + LOG.warn(MessageFormat.format( + JGitText.get().commitGraphChunkUnknown, + Integer.toHexString(chunkId))); + } + } + return builder.build(); + } + + private static class ChunkSegment { + final int id; + + final long offset; + + private ChunkSegment(int id, long offset) { + this.id = id; + this.offset = offset; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphV1.java new file mode 100644 index 000000000..da172192e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphV1.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; + +/** + * Support for the commit-graph v1 format. + * + * @see CommitGraph + */ +class CommitGraphV1 implements CommitGraph { + + private final GraphObjectIndex idx; + + private final GraphCommitData commitData; + + CommitGraphV1(GraphObjectIndex index, GraphCommitData commitData) { + this.idx = index; + this.commitData = commitData; + } + + /** {@inheritDoc} */ + @Override + public int findGraphPosition(AnyObjectId commit) { + return idx.findGraphPosition(commit); + } + + /** {@inheritDoc} */ + @Override + public CommitData getCommitData(int graphPos) { + if (graphPos < 0 || graphPos >= getCommitCnt()) { + return null; + } + return commitData.getCommitData(graphPos); + } + + /** {@inheritDoc} */ + @Override + public ObjectId getObjectId(int graphPos) { + return idx.getObjectId(graphPos); + } + + /** {@inheritDoc} */ + @Override + public long getCommitCnt() { + return idx.getCommitCnt(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/GraphCommitData.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/GraphCommitData.java new file mode 100644 index 000000000..6ae40ff55 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/GraphCommitData.java @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.COMMIT_DATA_WIDTH; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EDGE_LAST_MASK; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_EXTRA_EDGES_NEEDED; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_LAST_EDGE; +import static org.eclipse.jgit.internal.storage.commitgraph.CommitGraphConstants.GRAPH_NO_PARENT; + +import java.text.MessageFormat; +import java.util.Arrays; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph.CommitData; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.util.NB; + +/** + * Represent the collection of {@link CommitData}. + */ +class GraphCommitData { + + private static final int[] NO_PARENTS = {}; + + private final byte[] data; + + private final byte[] extraList; + + private final int hashLength; + + private final int commitDataLength; + + /** + * Initialize the GraphCommitData. + * + * @param hashLength + * length of object hash. + * @param commitData + * content of CommitData Chunk. + * @param extraList + * content of Extra Edge List Chunk. + */ + GraphCommitData(int hashLength, @NonNull byte[] commitData, + byte[] extraList) { + this.data = commitData; + this.extraList = extraList; + this.hashLength = hashLength; + this.commitDataLength = hashLength + COMMIT_DATA_WIDTH; + } + + /** + * Get the metadata of a commit。 + * + * @param graphPos + * the position in the commit-graph of the object. + * @return the metadata of a commit or null if not found. + */ + CommitData getCommitData(int graphPos) { + int dataIdx = commitDataLength * graphPos; + + // parse tree + ObjectId tree = ObjectId.fromRaw(data, dataIdx); + + // parse date + long dateHigh = NB.decodeUInt32(data, dataIdx + hashLength + 8) & 0x3; + long dateLow = NB.decodeUInt32(data, dataIdx + hashLength + 12); + long commitTime = dateHigh << 32 | dateLow; + + // parse generation + int generation = NB.decodeInt32(data, dataIdx + hashLength + 8) >> 2; + + // parse first parent + int parent1 = NB.decodeInt32(data, dataIdx + hashLength); + if (parent1 == GRAPH_NO_PARENT) { + return new CommitDataImpl(tree, NO_PARENTS, commitTime, generation); + } + + // parse second parent + int parent2 = NB.decodeInt32(data, dataIdx + hashLength + 4); + if (parent2 == GRAPH_NO_PARENT) { + return new CommitDataImpl(tree, new int[] { parent1 }, commitTime, + generation); + } + + if ((parent2 & GRAPH_EXTRA_EDGES_NEEDED) == 0) { + return new CommitDataImpl(tree, new int[] { parent1, parent2 }, + commitTime, generation); + } + + // parse parents for octopus merge + return new CommitDataImpl(tree, + findParentsForOctopusMerge(parent1, + parent2 & GRAPH_EDGE_LAST_MASK), + commitTime, generation); + } + + private int[] findParentsForOctopusMerge(int parent1, int extraEdgePos) { + int maxOffset = extraList.length - 4; + int offset = extraEdgePos * 4; + if (offset < 0 || offset > maxOffset) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().invalidExtraEdgeListPosition, + Integer.valueOf(extraEdgePos))); + } + int[] pList = new int[32]; + pList[0] = parent1; + int count = 1; + int parentPosition; + for (; offset <= maxOffset; offset += 4) { + if (count >= pList.length) { + // expand the pList + pList = Arrays.copyOf(pList, pList.length + 32); + } + parentPosition = NB.decodeInt32(extraList, offset); + if ((parentPosition & GRAPH_LAST_EDGE) != 0) { + pList[count++] = parentPosition & GRAPH_EDGE_LAST_MASK; + break; + } + pList[count++] = parentPosition; + } + return Arrays.copyOf(pList, count); + } + + private static class CommitDataImpl implements CommitData { + + private final ObjectId tree; + + private final int[] parents; + + private final long commitTime; + + private final int generation; + + public CommitDataImpl(ObjectId tree, int[] parents, long commitTime, + int generation) { + this.tree = tree; + this.parents = parents; + this.commitTime = commitTime; + this.generation = generation; + } + + @Override + public ObjectId getTree() { + return tree; + } + + @Override + public int[] getParents() { + return parents; + } + + @Override + public long getCommitTime() { + return commitTime; + } + + @Override + public int getGeneration() { + return generation; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/GraphObjectIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/GraphObjectIndex.java new file mode 100644 index 000000000..b0df46732 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/GraphObjectIndex.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.commitgraph; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.util.NB; + +/** + * The index which are used by the commit-graph to: + *

    + *
  • Get the object in commit-graph by using a specific position.
  • + *
  • Get the position of a specific object in commit-graph.
  • + *
+ */ +class GraphObjectIndex { + + private static final int FANOUT = 256; + + private final int hashLength; + + private final int[] fanoutTable; + + private final byte[] oidLookup; + + private final long commitCnt; + + /** + * Initialize the GraphObjectIndex. + * + * @param hashLength + * length of object hash. + * @param oidFanout + * content of OID Fanout Chunk. + * @param oidLookup + * content of OID Lookup Chunk. + * @throws CommitGraphFormatException + * commit-graph file's format is different from we expected. + */ + GraphObjectIndex(int hashLength, @NonNull byte[] oidFanout, + @NonNull byte[] oidLookup) throws CommitGraphFormatException { + this.hashLength = hashLength; + this.oidLookup = oidLookup; + + int[] table = new int[FANOUT]; + long uint32; + for (int k = 0; k < table.length; k++) { + uint32 = NB.decodeUInt32(oidFanout, k * 4); + if (table[k] > Integer.MAX_VALUE) { + throw new CommitGraphFormatException( + JGitText.get().commitGraphFileIsTooLargeForJgit); + } + table[k] = (int) uint32; + } + this.fanoutTable = table; + this.commitCnt = table[FANOUT - 1]; + } + + /** + * Find the position in the commit-graph of the specified id. + * + * @param id + * the id for which the commit-graph position will be found. + * @return the commit-graph position or -1 if the object was not found. + */ + int findGraphPosition(AnyObjectId id) { + int levelOne = id.getFirstByte(); + int high = fanoutTable[levelOne]; + int low = 0; + if (levelOne > 0) { + low = fanoutTable[levelOne - 1]; + } + do { + int mid = (low + high) >>> 1; + int pos = objIdOffset(mid); + int cmp = id.compareTo(oidLookup, pos); + if (cmp < 0) { + high = mid; + } else if (cmp == 0) { + return mid; + } else { + low = mid + 1; + } + } while (low < high); + return -1; + } + + /** + * Get the object at the commit-graph position. + * + * @param graphPos + * the position in the commit-graph of the object. + * @return the ObjectId or null if it's not found. + */ + ObjectId getObjectId(int graphPos) { + if (graphPos < 0 || graphPos >= commitCnt) { + return null; + } + return ObjectId.fromRaw(oidLookup, objIdOffset(graphPos)); + } + + long getCommitCnt() { + return commitCnt; + } + + private int objIdOffset(int pos) { + return hashLength * pos; + } +}