GC: Write commit-graph files when gc
If 'core.commitGraph' and 'gc.writeCommitGraph' are both true, then gc will rewrite the commit-graph file when 'git gc' is run. Defaults to false while the commit-graph feature matures. Bug: 574368 Change-Id: Ic94cd69034c524285c938414610f2e152198e06e Signed-off-by: kylezhao <kylezhao@tencent.com>
This commit is contained in:
parent
7016e2ddae
commit
b082c58e0f
|
@ -0,0 +1,154 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2022, Tencent.
|
||||||
|
*
|
||||||
|
* This program and the accompanying materials are made available under the
|
||||||
|
* terms of the Eclipse Distribution License v. 1.0 which is available at
|
||||||
|
* https://www.eclipse.org/org/documents/edl-v10.php.
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.eclipse.jgit.internal.storage.file;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertArrayEquals;
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.junit.TestRepository;
|
||||||
|
import org.eclipse.jgit.lib.ConfigConstants;
|
||||||
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
import org.eclipse.jgit.lib.StoredConfig;
|
||||||
|
import org.eclipse.jgit.revwalk.RevCommit;
|
||||||
|
import org.eclipse.jgit.util.IO;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class GcCommitGraphTest extends GcTestCase {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCommitGraphConfig() {
|
||||||
|
StoredConfig config = repo.getConfig();
|
||||||
|
assertFalse(gc.shouldWriteCommitGraphWhenGc());
|
||||||
|
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
|
||||||
|
assertTrue(gc.shouldWriteCommitGraphWhenGc());
|
||||||
|
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
|
||||||
|
assertFalse(gc.shouldWriteCommitGraphWhenGc());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWriteEmptyRepo() throws Exception {
|
||||||
|
StoredConfig config = repo.getConfig();
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
|
||||||
|
|
||||||
|
assertTrue(gc.shouldWriteCommitGraphWhenGc());
|
||||||
|
gc.writeCommitGraph(Collections.emptySet());
|
||||||
|
File graphFile = new File(repo.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
assertFalse(graphFile.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWriteWhenGc() throws Exception {
|
||||||
|
StoredConfig config = repo.getConfig();
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
|
||||||
|
|
||||||
|
RevCommit tip = commitChain(10);
|
||||||
|
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
|
||||||
|
bb.update(tip);
|
||||||
|
|
||||||
|
assertTrue(gc.shouldWriteCommitGraphWhenGc());
|
||||||
|
gc.gc();
|
||||||
|
File graphFile = new File(repo.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
assertGraphFile(graphFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDefaultWriteWhenGc() throws Exception {
|
||||||
|
RevCommit tip = commitChain(10);
|
||||||
|
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
|
||||||
|
bb.update(tip);
|
||||||
|
|
||||||
|
assertFalse(gc.shouldWriteCommitGraphWhenGc());
|
||||||
|
gc.gc();
|
||||||
|
File graphFile = new File(repo.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
assertFalse(graphFile.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDisableWriteWhenGc() throws Exception {
|
||||||
|
RevCommit tip = commitChain(10);
|
||||||
|
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
|
||||||
|
bb.update(tip);
|
||||||
|
File graphFile = new File(repo.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
|
||||||
|
StoredConfig config = repo.getConfig();
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, false);
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
|
||||||
|
|
||||||
|
gc.gc();
|
||||||
|
assertFalse(graphFile.exists());
|
||||||
|
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
|
||||||
|
gc.gc();
|
||||||
|
assertFalse(graphFile.exists());
|
||||||
|
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, false);
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
|
||||||
|
gc.gc();
|
||||||
|
assertFalse(graphFile.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWriteCommitGraphOnly() throws Exception {
|
||||||
|
RevCommit tip = commitChain(10);
|
||||||
|
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
|
||||||
|
bb.update(tip);
|
||||||
|
|
||||||
|
StoredConfig config = repo.getConfig();
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, false);
|
||||||
|
gc.writeCommitGraph(Collections.singleton(tip));
|
||||||
|
|
||||||
|
File graphFile = new File(repo.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
assertFalse(graphFile.exists());
|
||||||
|
|
||||||
|
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
|
||||||
|
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
|
||||||
|
gc.writeCommitGraph(Collections.singleton(tip));
|
||||||
|
assertGraphFile(graphFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertGraphFile(File graphFile) throws Exception {
|
||||||
|
assertTrue(graphFile.exists());
|
||||||
|
try (InputStream os = new FileInputStream(graphFile)) {
|
||||||
|
byte[] magic = new byte[4];
|
||||||
|
IO.readFully(os, magic, 0, 4);
|
||||||
|
assertArrayEquals(new byte[] { 'C', 'G', 'P', 'H' }, magic);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -63,10 +63,13 @@
|
||||||
import org.eclipse.jgit.errors.MissingObjectException;
|
import org.eclipse.jgit.errors.MissingObjectException;
|
||||||
import org.eclipse.jgit.errors.NoWorkTreeException;
|
import org.eclipse.jgit.errors.NoWorkTreeException;
|
||||||
import org.eclipse.jgit.internal.JGitText;
|
import org.eclipse.jgit.internal.JGitText;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter;
|
||||||
|
import org.eclipse.jgit.internal.storage.commitgraph.GraphCommits;
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackExt;
|
import org.eclipse.jgit.internal.storage.pack.PackExt;
|
||||||
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
import org.eclipse.jgit.internal.storage.pack.PackWriter;
|
||||||
import org.eclipse.jgit.lib.ConfigConstants;
|
import org.eclipse.jgit.lib.ConfigConstants;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
import org.eclipse.jgit.lib.CoreConfig;
|
||||||
import org.eclipse.jgit.lib.FileMode;
|
import org.eclipse.jgit.lib.FileMode;
|
||||||
import org.eclipse.jgit.lib.NullProgressMonitor;
|
import org.eclipse.jgit.lib.NullProgressMonitor;
|
||||||
import org.eclipse.jgit.lib.ObjectId;
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
|
@ -121,6 +124,8 @@ public class GC {
|
||||||
|
|
||||||
private static final int DEFAULT_AUTOLIMIT = 6700;
|
private static final int DEFAULT_AUTOLIMIT = 6700;
|
||||||
|
|
||||||
|
private static final boolean DEFAULT_WRITE_COMMIT_GRAPH = false;
|
||||||
|
|
||||||
private static volatile ExecutorService executor;
|
private static volatile ExecutorService executor;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -266,6 +271,9 @@ private Collection<Pack> doGc() throws IOException, ParseException {
|
||||||
Collection<Pack> newPacks = repack();
|
Collection<Pack> newPacks = repack();
|
||||||
prune(Collections.emptySet());
|
prune(Collections.emptySet());
|
||||||
// TODO: implement rerere_gc(pm);
|
// TODO: implement rerere_gc(pm);
|
||||||
|
if (shouldWriteCommitGraphWhenGc()) {
|
||||||
|
writeCommitGraph(refsToObjectIds(getAllRefs()));
|
||||||
|
}
|
||||||
return newPacks;
|
return newPacks;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -875,6 +883,102 @@ public Collection<Pack> repack() throws IOException {
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Set<ObjectId> refsToObjectIds(Collection<Ref> refs)
|
||||||
|
throws IOException {
|
||||||
|
Set<ObjectId> objectIds = new HashSet<>();
|
||||||
|
for (Ref ref : refs) {
|
||||||
|
checkCancelled();
|
||||||
|
if (ref.getPeeledObjectId() != null) {
|
||||||
|
objectIds.add(ref.getPeeledObjectId());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ref.getObjectId() != null) {
|
||||||
|
objectIds.add(ref.getObjectId());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return objectIds;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a new commit-graph file when 'core.commitGraph' is true.
|
||||||
|
*
|
||||||
|
* @param wants
|
||||||
|
* the list of wanted objects, writer walks commits starting at
|
||||||
|
* these. Must not be {@code null}.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
void writeCommitGraph(@NonNull Set<? extends ObjectId> wants)
|
||||||
|
throws IOException {
|
||||||
|
if (!repo.getConfig().get(CoreConfig.KEY).enableCommitGraph()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
checkCancelled();
|
||||||
|
if (wants.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
File tmpFile = null;
|
||||||
|
try (RevWalk walk = new RevWalk(repo)) {
|
||||||
|
CommitGraphWriter writer = new CommitGraphWriter(
|
||||||
|
GraphCommits.fromWalk(pm, wants, walk));
|
||||||
|
tmpFile = File.createTempFile("commit_", ".graph_tmp", //$NON-NLS-1$//$NON-NLS-2$
|
||||||
|
repo.getObjectDatabase().getInfoDirectory());
|
||||||
|
// write the commit-graph file
|
||||||
|
try (FileOutputStream fos = new FileOutputStream(tmpFile);
|
||||||
|
FileChannel channel = fos.getChannel();
|
||||||
|
OutputStream channelStream = Channels
|
||||||
|
.newOutputStream(channel)) {
|
||||||
|
writer.write(pm, channelStream);
|
||||||
|
channel.force(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// rename the temporary file to real file
|
||||||
|
File realFile = new File(repo.getObjectsDirectory(),
|
||||||
|
Constants.INFO_COMMIT_GRAPH);
|
||||||
|
FileUtils.rename(tmpFile, realFile, StandardCopyOption.ATOMIC_MOVE);
|
||||||
|
} finally {
|
||||||
|
if (tmpFile != null && tmpFile.exists()) {
|
||||||
|
tmpFile.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deleteTempCommitGraph();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void deleteTempCommitGraph() {
|
||||||
|
Path objectsDir = repo.getObjectDatabase().getInfoDirectory().toPath();
|
||||||
|
Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
|
||||||
|
if (!Files.exists(objectsDir)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (DirectoryStream<Path> stream = Files.newDirectoryStream(objectsDir,
|
||||||
|
"commit_*_tmp")) { //$NON-NLS-1$
|
||||||
|
stream.forEach(t -> {
|
||||||
|
try {
|
||||||
|
Instant lastModified = Files.getLastModifiedTime(t)
|
||||||
|
.toInstant();
|
||||||
|
if (lastModified.isBefore(threshold)) {
|
||||||
|
Files.deleteIfExists(t);
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.error(e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If {@code true}, will rewrite the commit-graph file when gc is run.
|
||||||
|
*
|
||||||
|
* @return true if commit-graph should be writen. Default is {@code false}.
|
||||||
|
*/
|
||||||
|
boolean shouldWriteCommitGraphWhenGc() {
|
||||||
|
return repo.getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION,
|
||||||
|
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH,
|
||||||
|
DEFAULT_WRITE_COMMIT_GRAPH);
|
||||||
|
}
|
||||||
|
|
||||||
private static boolean isHead(Ref ref) {
|
private static boolean isHead(Ref ref) {
|
||||||
return ref.getName().startsWith(Constants.R_HEADS);
|
return ref.getName().startsWith(Constants.R_HEADS);
|
||||||
}
|
}
|
||||||
|
|
|
@ -805,4 +805,8 @@ CachedObjectDirectory newCachedFileObjectDatabase() {
|
||||||
AlternateHandle.Id getAlternateId() {
|
AlternateHandle.Id getAlternateId() {
|
||||||
return handle.getId();
|
return handle.getId();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
File getInfoDirectory() {
|
||||||
|
return infoDirectory;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -284,6 +284,12 @@ public final class Constants {
|
||||||
*/
|
*/
|
||||||
public static final String INFO_HTTP_ALTERNATES = "info/http-alternates";
|
public static final String INFO_HTTP_ALTERNATES = "info/http-alternates";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* info commit-graph file (goes under OBJECTS)
|
||||||
|
* @since 6.5
|
||||||
|
*/
|
||||||
|
public static final String INFO_COMMIT_GRAPH = "info/commit-graph";
|
||||||
|
|
||||||
/** Packed refs file */
|
/** Packed refs file */
|
||||||
public static final String PACKED_REFS = "packed-refs";
|
public static final String PACKED_REFS = "packed-refs";
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue