CommitGraph: add commit-graph for FileObjectDatabase
This change makes JGit can read .git/objects/info/commit-graph file and then get CommitGraph. Loading a new commit-graph into memory requires additional time. After testing, loading a copy of the Linux's commit-graph(1039139 commits) is under 50ms. Bug: 574368 Change-Id: Iadfdd6ed437945d3cdfdbe988cf541198140a8bf Signed-off-by: kylezhao <kylezhao@tencent.com>
This commit is contained in:
parent
6722f25d56
commit
8a7348df69
|
@ -43,6 +43,7 @@
|
||||||
package org.eclipse.jgit.internal.storage.file;
|
package org.eclipse.jgit.internal.storage.file;
|
||||||
|
|
||||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertNull;
|
import static org.junit.Assert.assertNull;
|
||||||
import static org.junit.Assert.assertThrows;
|
import static org.junit.Assert.assertThrows;
|
||||||
|
@ -251,6 +252,50 @@ public void testShallowFileCorrupt() throws Exception {
|
||||||
IOException.class, () -> dir.getShallowCommits());
|
IOException.class, () -> dir.getShallowCommits());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetCommitGraph() throws Exception {
|
||||||
|
db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
|
||||||
|
db.getConfig().setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
|
||||||
|
|
||||||
|
// no commit-graph
|
||||||
|
ObjectDirectory dir = db.getObjectDatabase();
|
||||||
|
assertTrue(dir.getCommitGraph().isEmpty());
|
||||||
|
|
||||||
|
// add commit-graph
|
||||||
|
commitFile("file.txt", "content", "master");
|
||||||
|
GC gc = new GC(db);
|
||||||
|
gc.gc();
|
||||||
|
File file = new File(db.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
assertTrue(file.exists());
|
||||||
|
assertTrue(file.isFile());
|
||||||
|
assertTrue(dir.getCommitGraph().isPresent());
|
||||||
|
assertEquals(1, dir.getCommitGraph().get().getCommitCnt());
|
||||||
|
|
||||||
|
// update commit-graph
|
||||||
|
commitFile("file2.txt", "content", "master");
|
||||||
|
gc.gc();
|
||||||
|
assertEquals(2, dir.getCommitGraph().get().getCommitCnt());
|
||||||
|
|
||||||
|
// delete commit-graph
|
||||||
|
file.delete();
|
||||||
|
assertFalse(file.exists());
|
||||||
|
assertTrue(dir.getCommitGraph().isEmpty());
|
||||||
|
|
||||||
|
// commit-graph is corrupt
|
||||||
|
try (PrintWriter writer = new PrintWriter(file, UTF_8.name())) {
|
||||||
|
writer.println("this is a corrupt commit-graph");
|
||||||
|
}
|
||||||
|
assertTrue(dir.getCommitGraph().isEmpty());
|
||||||
|
|
||||||
|
// add commit-graph again
|
||||||
|
gc.gc();
|
||||||
|
assertTrue(dir.getCommitGraph().isPresent());
|
||||||
|
assertEquals(2, dir.getCommitGraph().get().getCommitCnt());
|
||||||
|
}
|
||||||
|
|
||||||
private Collection<Callable<ObjectId>> blobInsertersForTheSameFanOutDir(
|
private Collection<Callable<ObjectId>> blobInsertersForTheSameFanOutDir(
|
||||||
final ObjectDirectory dir) {
|
final ObjectDirectory dir) {
|
||||||
Callable<ObjectId> callable = () -> dir.newInserter()
|
Callable<ObjectId> callable = () -> dir.newInserter()
|
||||||
|
|
|
@ -164,6 +164,7 @@ connectionTimeOut=Connection time out: {0}
|
||||||
contextMustBeNonNegative=context must be >= 0
|
contextMustBeNonNegative=context must be >= 0
|
||||||
cookieFilePathRelative=git config http.cookieFile contains a relative path, should be absolute: {0}
|
cookieFilePathRelative=git config http.cookieFile contains a relative path, should be absolute: {0}
|
||||||
copyFileFailedNullFiles=Cannot copy file. Either origin or destination files are null
|
copyFileFailedNullFiles=Cannot copy file. Either origin or destination files are null
|
||||||
|
corruptCommitGraph=commit-graph file {0} is corrupt
|
||||||
corruptionDetectedReReadingAt=Corruption detected re-reading at {0}
|
corruptionDetectedReReadingAt=Corruption detected re-reading at {0}
|
||||||
corruptObjectBadDate=bad date
|
corruptObjectBadDate=bad date
|
||||||
corruptObjectBadEmail=bad email
|
corruptObjectBadEmail=bad email
|
||||||
|
@ -306,6 +307,7 @@ exceptionHookExecutionInterrupted=Execution of "{0}" hook interrupted.
|
||||||
exceptionOccurredDuringAddingOfOptionToALogCommand=Exception occurred during adding of {0} as option to a Log command
|
exceptionOccurredDuringAddingOfOptionToALogCommand=Exception occurred during adding of {0} as option to a Log command
|
||||||
exceptionOccurredDuringReadingOfGIT_DIR=Exception occurred during reading of $GIT_DIR/{0}. {1}
|
exceptionOccurredDuringReadingOfGIT_DIR=Exception occurred during reading of $GIT_DIR/{0}. {1}
|
||||||
exceptionWhileFindingUserHome=Problem determining the user home directory, trying Java user.home
|
exceptionWhileFindingUserHome=Problem determining the user home directory, trying Java user.home
|
||||||
|
exceptionWhileLoadingCommitGraph=Exception caught while loading commit-graph file {0}, the commit-graph file might be corrupt.
|
||||||
exceptionWhileReadingPack=Exception caught while accessing pack file {0}, the pack file might be corrupt. Caught {1} consecutive errors while trying to read this pack.
|
exceptionWhileReadingPack=Exception caught while accessing pack file {0}, the pack file might be corrupt. Caught {1} consecutive errors while trying to read this pack.
|
||||||
expectedACKNAKFoundEOF=Expected ACK/NAK, found EOF
|
expectedACKNAKFoundEOF=Expected ACK/NAK, found EOF
|
||||||
expectedACKNAKGot=Expected ACK/NAK, got: {0}
|
expectedACKNAKGot=Expected ACK/NAK, got: {0}
|
||||||
|
|
|
@ -192,6 +192,7 @@ public static JGitText get() {
|
||||||
/***/ public String contextMustBeNonNegative;
|
/***/ public String contextMustBeNonNegative;
|
||||||
/***/ public String cookieFilePathRelative;
|
/***/ public String cookieFilePathRelative;
|
||||||
/***/ public String copyFileFailedNullFiles;
|
/***/ public String copyFileFailedNullFiles;
|
||||||
|
/***/ public String corruptCommitGraph;
|
||||||
/***/ public String corruptionDetectedReReadingAt;
|
/***/ public String corruptionDetectedReReadingAt;
|
||||||
/***/ public String corruptObjectBadDate;
|
/***/ public String corruptObjectBadDate;
|
||||||
/***/ public String corruptObjectBadEmail;
|
/***/ public String corruptObjectBadEmail;
|
||||||
|
@ -334,6 +335,7 @@ public static JGitText get() {
|
||||||
/***/ public String exceptionOccurredDuringAddingOfOptionToALogCommand;
|
/***/ public String exceptionOccurredDuringAddingOfOptionToALogCommand;
|
||||||
/***/ public String exceptionOccurredDuringReadingOfGIT_DIR;
|
/***/ public String exceptionOccurredDuringReadingOfGIT_DIR;
|
||||||
/***/ public String exceptionWhileFindingUserHome;
|
/***/ public String exceptionWhileFindingUserHome;
|
||||||
|
/***/ public String exceptionWhileLoadingCommitGraph;
|
||||||
/***/ public String exceptionWhileReadingPack;
|
/***/ public String exceptionWhileReadingPack;
|
||||||
/***/ public String expectedACKNAKFoundEOF;
|
/***/ public String expectedACKNAKFoundEOF;
|
||||||
/***/ public String expectedACKNAKGot;
|
/***/ public String expectedACKNAKGot;
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle;
|
import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle;
|
||||||
|
@ -22,6 +23,7 @@
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||||
import org.eclipse.jgit.lib.AbbreviatedObjectId;
|
import org.eclipse.jgit.lib.AbbreviatedObjectId;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph;
|
||||||
import org.eclipse.jgit.lib.Config;
|
import org.eclipse.jgit.lib.Config;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
import org.eclipse.jgit.lib.ObjectDatabase;
|
import org.eclipse.jgit.lib.ObjectDatabase;
|
||||||
|
@ -259,6 +261,12 @@ Collection<Pack> getPacks() {
|
||||||
return wrapped.getPacks();
|
return wrapped.getPacks();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** {@inheritDoc} */
|
||||||
|
@Override
|
||||||
|
public Optional<CommitGraph> getCommitGraph() {
|
||||||
|
return wrapped.getCommitGraph();
|
||||||
|
}
|
||||||
|
|
||||||
private static class UnpackedObjectId extends ObjectIdOwnerMap.Entry {
|
private static class UnpackedObjectId extends ObjectIdOwnerMap.Entry {
|
||||||
UnpackedObjectId(AnyObjectId id) {
|
UnpackedObjectId(AnyObjectId id) {
|
||||||
super(id);
|
super(id);
|
||||||
|
|
|
@ -0,0 +1,135 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2022, Tencent.
|
||||||
|
*
|
||||||
|
* This program and the accompanying materials are made available under the
|
||||||
|
* terms of the Eclipse Distribution License v. 1.0 which is available at
|
||||||
|
* https://www.eclipse.org/org/documents/edl-v10.php.
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.storage.file;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.text.MessageFormat;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.annotations.NonNull;
|
||||||
|
import org.eclipse.jgit.internal.JGitText;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphFormatException;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphLoader;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph;
|
||||||
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Traditional file system for commit-graph.
|
||||||
|
* <p>
|
||||||
|
* This is the commit-graph file representation for a Git object database. Each
|
||||||
|
* call to {@link FileCommitGraph#get()} will recheck for newer versions.
|
||||||
|
*/
|
||||||
|
public class FileCommitGraph {
|
||||||
|
private final static Logger LOG = LoggerFactory
|
||||||
|
.getLogger(FileCommitGraph.class);
|
||||||
|
|
||||||
|
private final AtomicReference<GraphSnapshot> baseGraph;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize a reference to an on-disk commit-graph.
|
||||||
|
*
|
||||||
|
* @param objectsDir
|
||||||
|
* the location of the <code>objects</code> directory.
|
||||||
|
*/
|
||||||
|
FileCommitGraph(File objectsDir) {
|
||||||
|
this.baseGraph = new AtomicReference<>(new GraphSnapshot(
|
||||||
|
new File(objectsDir, Constants.INFO_COMMIT_GRAPH)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The method will first scan whether the ".git/objects/info/commit-graph"
|
||||||
|
* has been modified, if so, it will re-parse the file, otherwise it will
|
||||||
|
* return the same result as the last time.
|
||||||
|
*
|
||||||
|
* @return commit-graph or null if commit-graph file does not exist or
|
||||||
|
* corrupt.
|
||||||
|
*/
|
||||||
|
CommitGraph get() {
|
||||||
|
GraphSnapshot original = baseGraph.get();
|
||||||
|
synchronized (baseGraph) {
|
||||||
|
GraphSnapshot o, n;
|
||||||
|
do {
|
||||||
|
o = baseGraph.get();
|
||||||
|
if (o != original) {
|
||||||
|
// Another thread did the scan for us, while we
|
||||||
|
// were blocked on the monitor above.
|
||||||
|
//
|
||||||
|
return o.getCommitGraph();
|
||||||
|
}
|
||||||
|
n = o.refresh();
|
||||||
|
if (n == o) {
|
||||||
|
return n.getCommitGraph();
|
||||||
|
}
|
||||||
|
} while (!baseGraph.compareAndSet(o, n));
|
||||||
|
return n.getCommitGraph();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class GraphSnapshot {
|
||||||
|
private final File file;
|
||||||
|
|
||||||
|
private final FileSnapshot snapshot;
|
||||||
|
|
||||||
|
private final CommitGraph graph;
|
||||||
|
|
||||||
|
GraphSnapshot(@NonNull File file) {
|
||||||
|
this(file, FileSnapshot.save(file), null);
|
||||||
|
}
|
||||||
|
|
||||||
|
GraphSnapshot(@NonNull File file, @NonNull FileSnapshot snapshot,
|
||||||
|
CommitGraph graph) {
|
||||||
|
this.file = file;
|
||||||
|
this.snapshot = snapshot;
|
||||||
|
this.graph = graph;
|
||||||
|
}
|
||||||
|
|
||||||
|
CommitGraph getCommitGraph() {
|
||||||
|
return graph;
|
||||||
|
}
|
||||||
|
|
||||||
|
GraphSnapshot refresh() {
|
||||||
|
if (graph == null && !file.exists()) {
|
||||||
|
// commit-graph file didn't exist
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
if (!snapshot.isModified(file)) {
|
||||||
|
// commit-graph file was not modified
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
return new GraphSnapshot(file, FileSnapshot.save(file), open(file));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CommitGraph open(File file) {
|
||||||
|
try {
|
||||||
|
return CommitGraphLoader.open(file);
|
||||||
|
} catch (FileNotFoundException noFile) {
|
||||||
|
// ignore if file do not exist
|
||||||
|
return null;
|
||||||
|
} catch (IOException e) {
|
||||||
|
if (e instanceof CommitGraphFormatException) {
|
||||||
|
LOG.warn(
|
||||||
|
MessageFormat.format(
|
||||||
|
JGitText.get().corruptCommitGraph, file),
|
||||||
|
e);
|
||||||
|
} else {
|
||||||
|
LOG.error(MessageFormat.format(
|
||||||
|
JGitText.get().exceptionWhileLoadingCommitGraph,
|
||||||
|
file), e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,8 +13,10 @@
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph;
|
||||||
import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
|
import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||||
import org.eclipse.jgit.lib.AbbreviatedObjectId;
|
import org.eclipse.jgit.lib.AbbreviatedObjectId;
|
||||||
|
@ -72,4 +74,6 @@ abstract InsertLooseObjectResult insertUnpackedObject(File tmp,
|
||||||
abstract Pack openPack(File pack) throws IOException;
|
abstract Pack openPack(File pack) throws IOException;
|
||||||
|
|
||||||
abstract Collection<Pack> getPacks();
|
abstract Collection<Pack> getPacks();
|
||||||
|
|
||||||
|
abstract Optional<CommitGraph> getCommitGraph();
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,6 +28,7 @@
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
|
@ -37,6 +38,7 @@
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||||
import org.eclipse.jgit.lib.AbbreviatedObjectId;
|
import org.eclipse.jgit.lib.AbbreviatedObjectId;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph;
|
||||||
import org.eclipse.jgit.lib.Config;
|
import org.eclipse.jgit.lib.Config;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
import org.eclipse.jgit.lib.ObjectDatabase;
|
import org.eclipse.jgit.lib.ObjectDatabase;
|
||||||
|
@ -85,6 +87,8 @@ public class ObjectDirectory extends FileObjectDatabase {
|
||||||
|
|
||||||
private final File alternatesFile;
|
private final File alternatesFile;
|
||||||
|
|
||||||
|
private final FileCommitGraph fileCommitGraph;
|
||||||
|
|
||||||
private final FS fs;
|
private final FS fs;
|
||||||
|
|
||||||
private final AtomicReference<AlternateHandle[]> alternates;
|
private final AtomicReference<AlternateHandle[]> alternates;
|
||||||
|
@ -124,6 +128,7 @@ public ObjectDirectory(final Config cfg, final File dir,
|
||||||
loose = new LooseObjects(objects);
|
loose = new LooseObjects(objects);
|
||||||
packed = new PackDirectory(config, packDirectory);
|
packed = new PackDirectory(config, packDirectory);
|
||||||
preserved = new PackDirectory(config, preservedDirectory);
|
preserved = new PackDirectory(config, preservedDirectory);
|
||||||
|
fileCommitGraph = new FileCommitGraph(objects);
|
||||||
this.fs = fs;
|
this.fs = fs;
|
||||||
this.shallowFile = shallowFile;
|
this.shallowFile = shallowFile;
|
||||||
|
|
||||||
|
@ -227,6 +232,12 @@ public long getApproximateObjectCount() {
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** {@inheritDoc} */
|
||||||
|
@Override
|
||||||
|
public Optional<CommitGraph> getCommitGraph() {
|
||||||
|
return Optional.ofNullable(fileCommitGraph.get());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
* <p>
|
* <p>
|
||||||
|
|
Loading…
Reference in New Issue