motiejus
/
jgit
1
Fork 0
Code Releases Activity

GC: Write commit-graph files when gc 

If 'core.commitGraph' and 'gc.writeCommitGraph' are both true, then gc
will rewrite the commit-graph file when 'git gc' is run. Defaults to
false while the commit-graph feature matures.

Bug: 574368
Change-Id: Ic94cd69034c524285c938414610f2e152198e06e
Signed-off-by: kylezhao <kylezhao@tencent.com>
This commit is contained in:
kylezhao 2021-07-12 17:07:13 +08:00 committed by Ivan Frade
parent 7016e2ddae
commit b082c58e0f
4 changed files with 268 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java Normal file
View File

@ -0,0 +1,154 @@
/*
* Copyright (C) 2022, Tencent.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.internal.storage.file;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Collections;
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.util.IO;
import org.junit.Test;
public class GcCommitGraphTest extends GcTestCase {
@Test
public void testCommitGraphConfig() {
StoredConfig config = repo.getConfig();
assertFalse(gc.shouldWriteCommitGraphWhenGc());
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
assertTrue(gc.shouldWriteCommitGraphWhenGc());
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
assertFalse(gc.shouldWriteCommitGraphWhenGc());
}
@Test
public void testWriteEmptyRepo() throws Exception {
StoredConfig config = repo.getConfig();
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
assertTrue(gc.shouldWriteCommitGraphWhenGc());
gc.writeCommitGraph(Collections.emptySet());
File graphFile = new File(repo.getObjectsDirectory(),
Constants.INFO_COMMIT_GRAPH);
assertFalse(graphFile.exists());
}
@Test
public void testWriteWhenGc() throws Exception {
StoredConfig config = repo.getConfig();
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
RevCommit tip = commitChain(10);
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
bb.update(tip);
assertTrue(gc.shouldWriteCommitGraphWhenGc());
gc.gc();
File graphFile = new File(repo.getObjectsDirectory(),
Constants.INFO_COMMIT_GRAPH);
assertGraphFile(graphFile);
}
@Test
public void testDefaultWriteWhenGc() throws Exception {
RevCommit tip = commitChain(10);
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
bb.update(tip);
assertFalse(gc.shouldWriteCommitGraphWhenGc());
gc.gc();
File graphFile = new File(repo.getObjectsDirectory(),
Constants.INFO_COMMIT_GRAPH);
assertFalse(graphFile.exists());
}
@Test
public void testDisableWriteWhenGc() throws Exception {
RevCommit tip = commitChain(10);
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
bb.update(tip);
File graphFile = new File(repo.getObjectsDirectory(),
Constants.INFO_COMMIT_GRAPH);
StoredConfig config = repo.getConfig();
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, false);
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true);
gc.gc();
assertFalse(graphFile.exists());
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
gc.gc();
assertFalse(graphFile.exists());
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, false);
config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false);
gc.gc();
assertFalse(graphFile.exists());
}
@Test
public void testWriteCommitGraphOnly() throws Exception {
RevCommit tip = commitChain(10);
TestRepository.BranchBuilder bb = tr.branch("refs/heads/master");
bb.update(tip);
StoredConfig config = repo.getConfig();
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, false);
gc.writeCommitGraph(Collections.singleton(tip));
File graphFile = new File(repo.getObjectsDirectory(),
Constants.INFO_COMMIT_GRAPH);
assertFalse(graphFile.exists());
config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
ConfigConstants.CONFIG_COMMIT_GRAPH, true);
gc.writeCommitGraph(Collections.singleton(tip));
assertGraphFile(graphFile);
}
private void assertGraphFile(File graphFile) throws Exception {
assertTrue(graphFile.exists());
try (InputStream os = new FileInputStream(graphFile)) {
byte[] magic = new byte[4];
IO.readFully(os, magic, 0, 4);
assertArrayEquals(new byte[] { 'C', 'G', 'P', 'H' }, magic);
}
}
}

104
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
View File

@ -63,10 +63,13 @@
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.NoWorkTreeException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter;
import org.eclipse.jgit.internal.storage.commitgraph.GraphCommits;
import org.eclipse.jgit.internal.storage.pack.PackExt;
import org.eclipse.jgit.internal.storage.pack.PackWriter;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.CoreConfig;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
@ -121,6 +124,8 @@ public class GC {
private static final int DEFAULT_AUTOLIMIT = 6700;
private static final boolean DEFAULT_WRITE_COMMIT_GRAPH = false;
private static volatile ExecutorService executor;
/**
@ -266,6 +271,9 @@ private Collection<Pack> doGc() throws IOException, ParseException {
Collection<Pack> newPacks = repack();
prune(Collections.emptySet());
// TODO: implement rerere_gc(pm);
if (shouldWriteCommitGraphWhenGc()) {
writeCommitGraph(refsToObjectIds(getAllRefs()));
}
return newPacks;
}
@ -875,6 +883,102 @@ public Collection<Pack> repack() throws IOException {
return ret;
}
private Set<ObjectId> refsToObjectIds(Collection<Ref> refs)
throws IOException {
Set<ObjectId> objectIds = new HashSet<>();
for (Ref ref : refs) {
checkCancelled();
if (ref.getPeeledObjectId() != null) {
objectIds.add(ref.getPeeledObjectId());
continue;
}
if (ref.getObjectId() != null) {
objectIds.add(ref.getObjectId());
}
}
return objectIds;
}
/**
* Generate a new commit-graph file when 'core.commitGraph' is true.
*
* @param wants
* the list of wanted objects, writer walks commits starting at
* these. Must not be {@code null}.
* @throws IOException
*/
void writeCommitGraph(@NonNull Set<? extends ObjectId> wants)
throws IOException {
if (!repo.getConfig().get(CoreConfig.KEY).enableCommitGraph()) {
return;
}
checkCancelled();
if (wants.isEmpty()) {
return;
}
File tmpFile = null;
try (RevWalk walk = new RevWalk(repo)) {
CommitGraphWriter writer = new CommitGraphWriter(
GraphCommits.fromWalk(pm, wants, walk));
tmpFile = File.createTempFile("commit_", ".graph_tmp", //$NON-NLS-1$//$NON-NLS-2$
repo.getObjectDatabase().getInfoDirectory());
// write the commit-graph file
try (FileOutputStream fos = new FileOutputStream(tmpFile);
FileChannel channel = fos.getChannel();
OutputStream channelStream = Channels
.newOutputStream(channel)) {
writer.write(pm, channelStream);
channel.force(true);
}
// rename the temporary file to real file
File realFile = new File(repo.getObjectsDirectory(),
Constants.INFO_COMMIT_GRAPH);
FileUtils.rename(tmpFile, realFile, StandardCopyOption.ATOMIC_MOVE);
} finally {
if (tmpFile != null && tmpFile.exists()) {
tmpFile.delete();
}
}
deleteTempCommitGraph();
}
private void deleteTempCommitGraph() {
Path objectsDir = repo.getObjectDatabase().getInfoDirectory().toPath();
Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
if (!Files.exists(objectsDir)) {
return;
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(objectsDir,
"commit_*_tmp")) { //$NON-NLS-1$
stream.forEach(t -> {
try {
Instant lastModified = Files.getLastModifiedTime(t)
.toInstant();
if (lastModified.isBefore(threshold)) {
Files.deleteIfExists(t);
}
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
});
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
/**
* If {@code true}, will rewrite the commit-graph file when gc is run.
*
* @return true if commit-graph should be writen. Default is {@code false}.
*/
boolean shouldWriteCommitGraphWhenGc() {
return repo.getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION,
ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH,
DEFAULT_WRITE_COMMIT_GRAPH);
}
private static boolean isHead(Ref ref) {
return ref.getName().startsWith(Constants.R_HEADS);
}

4
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
View File

@ -805,4 +805,8 @@ CachedObjectDirectory newCachedFileObjectDatabase() {
AlternateHandle.Id getAlternateId() {
return handle.getId();
}
File getInfoDirectory() {
return infoDirectory;
}
}

6
org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java
View File

@ -284,6 +284,12 @@ public final class Constants {
*/
public static final String INFO_HTTP_ALTERNATES = "info/http-alternates";
/**
* info commit-graph file (goes under OBJECTS)
* @since 6.5
*/
public static final String INFO_COMMIT_GRAPH = "info/commit-graph";
/** Packed refs file */
public static final String PACKED_REFS = "packed-refs";