From b082c58e0ff3e829071e90b47df022e77cd3dea2 Mon Sep 17 00:00:00 2001 From: kylezhao Date: Mon, 12 Jul 2021 17:07:13 +0800 Subject: [PATCH] GC: Write commit-graph files when gc If 'core.commitGraph' and 'gc.writeCommitGraph' are both true, then gc will rewrite the commit-graph file when 'git gc' is run. Defaults to false while the commit-graph feature matures. Bug: 574368 Change-Id: Ic94cd69034c524285c938414610f2e152198e06e Signed-off-by: kylezhao --- .../storage/file/GcCommitGraphTest.java | 154 ++++++++++++++++++ .../jgit/internal/storage/file/GC.java | 104 ++++++++++++ .../storage/file/ObjectDirectory.java | 4 + .../src/org/eclipse/jgit/lib/Constants.java | 6 + 4 files changed, 268 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java new file mode 100644 index 000000000..0a0d85c8a --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcCommitGraphTest.java @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2022, Tencent. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Collections; + +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.IO; +import org.junit.Test; + +public class GcCommitGraphTest extends GcTestCase { + + @Test + public void testCommitGraphConfig() { + StoredConfig config = repo.getConfig(); + assertFalse(gc.shouldWriteCommitGraphWhenGc()); + + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true); + assertTrue(gc.shouldWriteCommitGraphWhenGc()); + + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false); + assertFalse(gc.shouldWriteCommitGraphWhenGc()); + } + + @Test + public void testWriteEmptyRepo() throws Exception { + StoredConfig config = repo.getConfig(); + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, true); + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true); + + assertTrue(gc.shouldWriteCommitGraphWhenGc()); + gc.writeCommitGraph(Collections.emptySet()); + File graphFile = new File(repo.getObjectsDirectory(), + Constants.INFO_COMMIT_GRAPH); + assertFalse(graphFile.exists()); + } + + @Test + public void testWriteWhenGc() throws Exception { + StoredConfig config = repo.getConfig(); + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, true); + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true); + + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/master"); + bb.update(tip); + + assertTrue(gc.shouldWriteCommitGraphWhenGc()); + gc.gc(); + File graphFile = new File(repo.getObjectsDirectory(), + Constants.INFO_COMMIT_GRAPH); + assertGraphFile(graphFile); + } + + @Test + public void testDefaultWriteWhenGc() throws Exception { + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/master"); + bb.update(tip); + + assertFalse(gc.shouldWriteCommitGraphWhenGc()); + gc.gc(); + File graphFile = new File(repo.getObjectsDirectory(), + Constants.INFO_COMMIT_GRAPH); + assertFalse(graphFile.exists()); + } + + @Test + public void testDisableWriteWhenGc() throws Exception { + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/master"); + bb.update(tip); + File graphFile = new File(repo.getObjectsDirectory(), + Constants.INFO_COMMIT_GRAPH); + + StoredConfig config = repo.getConfig(); + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, false); + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, true); + + gc.gc(); + assertFalse(graphFile.exists()); + + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, true); + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false); + gc.gc(); + assertFalse(graphFile.exists()); + + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, false); + config.setBoolean(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, false); + gc.gc(); + assertFalse(graphFile.exists()); + } + + @Test + public void testWriteCommitGraphOnly() throws Exception { + RevCommit tip = commitChain(10); + TestRepository.BranchBuilder bb = tr.branch("refs/heads/master"); + bb.update(tip); + + StoredConfig config = repo.getConfig(); + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, false); + gc.writeCommitGraph(Collections.singleton(tip)); + + File graphFile = new File(repo.getObjectsDirectory(), + Constants.INFO_COMMIT_GRAPH); + assertFalse(graphFile.exists()); + + config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_COMMIT_GRAPH, true); + gc.writeCommitGraph(Collections.singleton(tip)); + assertGraphFile(graphFile); + } + + private void assertGraphFile(File graphFile) throws Exception { + assertTrue(graphFile.exists()); + try (InputStream os = new FileInputStream(graphFile)) { + byte[] magic = new byte[4]; + IO.readFully(os, magic, 0, 4); + assertArrayEquals(new byte[] { 'C', 'G', 'P', 'H' }, magic); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index d81f8ee49..c9ecebe80 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -63,10 +63,13 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.NoWorkTreeException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter; +import org.eclipse.jgit.internal.storage.commitgraph.GraphCommits; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -121,6 +124,8 @@ public class GC { private static final int DEFAULT_AUTOLIMIT = 6700; + private static final boolean DEFAULT_WRITE_COMMIT_GRAPH = false; + private static volatile ExecutorService executor; /** @@ -266,6 +271,9 @@ private Collection doGc() throws IOException, ParseException { Collection newPacks = repack(); prune(Collections.emptySet()); // TODO: implement rerere_gc(pm); + if (shouldWriteCommitGraphWhenGc()) { + writeCommitGraph(refsToObjectIds(getAllRefs())); + } return newPacks; } @@ -875,6 +883,102 @@ public Collection repack() throws IOException { return ret; } + private Set refsToObjectIds(Collection refs) + throws IOException { + Set objectIds = new HashSet<>(); + for (Ref ref : refs) { + checkCancelled(); + if (ref.getPeeledObjectId() != null) { + objectIds.add(ref.getPeeledObjectId()); + continue; + } + + if (ref.getObjectId() != null) { + objectIds.add(ref.getObjectId()); + } + } + return objectIds; + } + + /** + * Generate a new commit-graph file when 'core.commitGraph' is true. + * + * @param wants + * the list of wanted objects, writer walks commits starting at + * these. Must not be {@code null}. + * @throws IOException + */ + void writeCommitGraph(@NonNull Set wants) + throws IOException { + if (!repo.getConfig().get(CoreConfig.KEY).enableCommitGraph()) { + return; + } + checkCancelled(); + if (wants.isEmpty()) { + return; + } + File tmpFile = null; + try (RevWalk walk = new RevWalk(repo)) { + CommitGraphWriter writer = new CommitGraphWriter( + GraphCommits.fromWalk(pm, wants, walk)); + tmpFile = File.createTempFile("commit_", ".graph_tmp", //$NON-NLS-1$//$NON-NLS-2$ + repo.getObjectDatabase().getInfoDirectory()); + // write the commit-graph file + try (FileOutputStream fos = new FileOutputStream(tmpFile); + FileChannel channel = fos.getChannel(); + OutputStream channelStream = Channels + .newOutputStream(channel)) { + writer.write(pm, channelStream); + channel.force(true); + } + + // rename the temporary file to real file + File realFile = new File(repo.getObjectsDirectory(), + Constants.INFO_COMMIT_GRAPH); + FileUtils.rename(tmpFile, realFile, StandardCopyOption.ATOMIC_MOVE); + } finally { + if (tmpFile != null && tmpFile.exists()) { + tmpFile.delete(); + } + } + deleteTempCommitGraph(); + } + + private void deleteTempCommitGraph() { + Path objectsDir = repo.getObjectDatabase().getInfoDirectory().toPath(); + Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS); + if (!Files.exists(objectsDir)) { + return; + } + try (DirectoryStream stream = Files.newDirectoryStream(objectsDir, + "commit_*_tmp")) { //$NON-NLS-1$ + stream.forEach(t -> { + try { + Instant lastModified = Files.getLastModifiedTime(t) + .toInstant(); + if (lastModified.isBefore(threshold)) { + Files.deleteIfExists(t); + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + }); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + + /** + * If {@code true}, will rewrite the commit-graph file when gc is run. + * + * @return true if commit-graph should be writen. Default is {@code false}. + */ + boolean shouldWriteCommitGraphWhenGc() { + return repo.getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_WRITE_COMMIT_GRAPH, + DEFAULT_WRITE_COMMIT_GRAPH); + } + private static boolean isHead(Ref ref) { return ref.getName().startsWith(Constants.R_HEADS); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index ff7ef9327..f7ccceca4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -805,4 +805,8 @@ CachedObjectDirectory newCachedFileObjectDatabase() { AlternateHandle.Id getAlternateId() { return handle.getId(); } + + File getInfoDirectory() { + return infoDirectory; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index c166abe12..0b8bf8c6c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -284,6 +284,12 @@ public final class Constants { */ public static final String INFO_HTTP_ALTERNATES = "info/http-alternates"; + /** + * info commit-graph file (goes under OBJECTS) + * @since 6.5 + */ + public static final String INFO_COMMIT_GRAPH = "info/commit-graph"; + /** Packed refs file */ public static final String PACKED_REFS = "packed-refs";