motiejus
/
jgit
1
Fork 0
Code Releases Activity

Merge branch 'stable-6.2' into stable-6.3 

* stable-6.2:
  Shortcut during git fetch for avoiding looping through all local refs
  FetchCommand: fix fetchSubmodules to work on a Ref to a blob
  Silence API warnings introduced by I466dcde6
  Allow the exclusions of refs prefixes from bitmap
  PackWriterBitmapPreparer: do not include annotated tags in bitmap
  BatchingProgressMonitor: avoid int overflow when computing percentage
  Speedup GC listing objects referenced from reflogs
  FileSnapshotTest: Add more MISSING_FILE coverage

Change-Id: I2ff386d9a096277360e6c7bd5535b49984620fb3
This commit is contained in:
Matthias Sohn 2023-02-01 00:59:32 +01:00
parent 5bd2832134 d8c02aec6a
commit 82e1362e07
14 changed files with 300 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Documentation/config-options.md
View File

@ -87,6 +87,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS
| `pack.bitmapContiguousCommitCount` | `100` | ⃞ | Count of most recent commits for which to build bitmaps. |
| `pack.bitmapDistantCommitSpan` | `5000` | ⃞ | Span of commits when building bitmaps for distant history. |
| `pack.bitmapExcessiveBranchCount` | `100` | ⃞ | The count of branches deemed "excessive". If the count of branches in a repository exceeds this number and bitmaps are enabled, "inactive" branches will have fewer bitmaps than "active" branches. |
| `pack.bitmapExcludedRefsPrefixes` | | ⃞ | The refs prefixes to be excluded when building bitmaps. May be specified more than once to exclude multiple prefixes. |
| `pack.bitmapInactiveBranchAgeInDays` | `90` | ⃞ | Age in days that marks a branch as "inactive" for bitmap creation. |
| `pack.bitmapRecentCommitCount` | `20000` | ⃞ | Count at which to switch from `bitmapRecentCommitSpan` to `bitmapDistantCommitSpan`. |
| `pack.bitmapRecentCommitSpan` | `100` | ⃞ | Span of commits when building bitmaps for recent history. |

14
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java
View File

@ -209,6 +209,20 @@ public void fileSnapshotEquals() throws Exception {
assertTrue(fs2.equals(fs1));
}
@Test
public void snapshotAndFileMissingIsNotModified() throws Exception {
File doesNotExist = trash.resolve("DOES_NOT_EXIST").toFile();
FileSnapshot missing = FileSnapshot.save(doesNotExist);
assertFalse(missing.isModified(doesNotExist));
}
@Test
public void missingFileEquals() throws Exception {
FileSnapshot missing = FileSnapshot.save(
trash.resolve("DOES_NOT_EXIST").toFile());
assertTrue(missing.equals(FileSnapshot.MISSING_FILE));
}
@SuppressWarnings("boxing")
@Test
public void detectFileModified() throws IOException {

52
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
View File

@ -82,6 +82,40 @@ private void testBitmapSpansNoMerges(boolean withTags) throws Exception {
}
}
@Test
public void testBitmapDoesNotIncludeAnnotatedTags() throws Exception {
/*
* Make sure that the bitmap generated for the following commit
* graph does not include commit2 because it is not reachable by any
* heads, despite being reachable from tag1 through the annotated-tag1.
*
* refs/heads/main
* ^
* |
* commit1 <-- commit2 <- annotated-tag1 <- tag1
* ^
* |
* commit0
*/
String mainBranch = "refs/heads/main";
BranchBuilder bb = tr.branch(mainBranch);
String commitMsg = "commit msg";
String fileBody = "file body";
String tagName = "tag1";
bb.commit().message(commitMsg + " 1").add("file1", fileBody).create();
RevCommit commit1 = bb.commit().message(commitMsg + " 2").add("file2", fileBody).create();
RevCommit commit2 = bb.commit().message(commitMsg + " 3").add("file3", fileBody).create();
tr.lightweightTag(tagName, tr.tag(tagName, commit2));
tr.branch(mainBranch).update(commit1);
gc.setExpireAgeMillis(0);
gc.gc();
// Create only 2 bitmaps, for commit0 and commit1, excluding commit2
assertEquals(2, gc.getStatistics().numberOfBitmaps);
}
@Test
public void testBitmapSpansWithMerges() throws Exception {
/*
@ -186,6 +220,24 @@ public void testBitmapsForExcessiveBranches() throws Exception {
gc.getStatistics().numberOfBitmaps);
}
@Test
public void testBitmapsForExcludedBranches() throws Exception {
createNewCommitOnNewBranch("main");
createNewCommitOnNewBranch("other");
PackConfig packConfig = new PackConfig();
packConfig.setBitmapExcludedRefsPrefixes(new String[] { "refs/heads/other" });
gc.setPackConfig(packConfig);
gc.gc();
assertEquals(1,
gc.getStatistics().numberOfBitmaps);
}
private void createNewCommitOnNewBranch(String branchName) throws Exception {
BranchBuilder bb = tr.branch("refs/heads/" + branchName);
String msg = "New branch " + branchName;
bb.commit().message(msg).add("some-filename.txt", msg).create();
}
@Test
public void testSelectionOrderingWithChains() throws Exception {
/*-

83
org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TextProgressMonitorTest.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright (C) 2023, SAP SE or an SAP affiliate company and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.lib;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class TextProgressMonitorTest {
private TextProgressMonitor m;
private ByteArrayOutputStream buf;
@Before
public void setup() {
buf = new ByteArrayOutputStream();
m = new TextProgressMonitor(
new OutputStreamWriter(buf, StandardCharsets.UTF_8));
}
@Test
public void testSimple() throws Exception {
m.beginTask("task", 10);
for (int i = 0; i < 10; i++) {
m.update(1);
}
m.endTask();
Assert.assertArrayEquals(
new String[] { "", "task: 10% ( 1/10)",
"task: 20% ( 2/10)",
"task: 30% ( 3/10)",
"task: 40% ( 4/10)",
"task: 50% ( 5/10)",
"task: 60% ( 6/10)",
"task: 70% ( 7/10)",
"task: 80% ( 8/10)",
"task: 90% ( 9/10)",
"task: 100% (10/10)",
"task: 100% (10/10)\n" },
bufLines());
}
@Test
public void testLargeNumbers() throws Exception {
m.beginTask("task", 1_000_000_000);
for (int i = 0; i < 10; i++) {
m.update(100_000_000);
}
m.endTask();
Assert.assertArrayEquals(
new String[] { "",
"task: 10% ( 100000000/1000000000)",
"task: 20% ( 200000000/1000000000)",
"task: 30% ( 300000000/1000000000)",
"task: 40% ( 400000000/1000000000)",
"task: 50% ( 500000000/1000000000)",
"task: 60% ( 600000000/1000000000)",
"task: 70% ( 700000000/1000000000)",
"task: 80% ( 800000000/1000000000)",
"task: 90% ( 900000000/1000000000)",
"task: 100% (1000000000/1000000000)",
"task: 100% (1000000000/1000000000)\n" },
bufLines());
}
String[] bufLines() throws UnsupportedEncodingException {
String s = new String(buf.toString(StandardCharsets.UTF_8.name()));
return s.split("\r");
}
}

40
org.eclipse.jgit/.settings/.api_filters
View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<component id="org.eclipse.jgit" version="2">
<resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants">
<filter id="1142947843">
<message_arguments>
<message_argument value="5.13.2"/>
<message_argument value="CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.13.2"/>
@ -22,6 +28,14 @@
</message_arguments>
</filter>
</resource>
<resource path="src/org/eclipse/jgit/lib/Repository.java" type="org.eclipse.jgit.lib.Repository">
<filter id="1142947843">
<message_arguments>
<message_argument value="5.13.2"/>
<message_argument value="getReflogReader(Ref)"/>
</message_arguments>
</filter>
</resource>
<resource path="src/org/eclipse/jgit/merge/ResolveMerger.java" type="org.eclipse.jgit.merge.ResolveMerger">
<filter id="336658481">
<message_arguments>
@ -80,6 +94,32 @@
</message_arguments>
</filter>
</resource>
<resource path="src/org/eclipse/jgit/storage/pack/PackConfig.java" type="org.eclipse.jgit.storage.pack.PackConfig">
<filter id="336658481">
<message_arguments>
<message_argument value="org.eclipse.jgit.storage.pack.PackConfig"/>
<message_argument value="DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.13.2"/>
<message_argument value="DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.13.2"/>
<message_argument value="getBitmapExcludedRefsPrefixes()"/>
</message_arguments>
</filter>
<filter id="1142947843">
<message_arguments>
<message_argument value="5.13.2"/>
<message_argument value="setBitmapExcludedRefsPrefixes(String[])"/>
</message_arguments>
</filter>
</resource>
<resource path="src/org/eclipse/jgit/transport/AwsRequestSignerV4.java" type="org.eclipse.jgit.transport.AwsRequestSignerV4">
<filter id="1109393411">
<message_arguments>

3
org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java
View File

@ -151,6 +151,9 @@ private void fetchSubmodules(FetchResult results)
if (fetchHead == null) {
return;
}
if (revWalk.parseAny(fetchHead).getType() == Constants.OBJ_BLOB) {
return;
}
walk.setTree(revWalk.parseTree(fetchHead));
while (walk.next()) {
try (Repository submoduleRepo = walk.getRepository()) {

7
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java
View File

@ -31,6 +31,7 @@
import java.util.Objects;
import java.util.Set;
import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.annotations.Nullable;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.attributes.AttributesNode;
@ -530,6 +531,12 @@ public ReflogReader getReflogReader(String refName) throws IOException {
return new ReflogReaderImpl(this, ref.getName());
}
@Override
public @NonNull ReflogReader getReflogReader(@NonNull Ref ref)
throws IOException {
return new ReflogReaderImpl(this, ref.getName());
}
/** {@inheritDoc} */
@Override
public AttributesNodeProvider createAttributesNodeProvider() {

28
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
View File

@ -794,6 +794,10 @@ public Collection<Pack> repack() throws IOException {
Set<ObjectId> tagTargets = new HashSet<>();
Set<ObjectId> indexObjects = listNonHEADIndexObjects();
Set<ObjectId> refsToExcludeFromBitmap = repo.getRefDatabase()
.getRefsByPrefix(pconfig.getBitmapExcludedRefsPrefixes())
.stream().map(Ref::getObjectId).collect(Collectors.toSet());
for (Ref ref : refsBefore) {
checkCancelled();
nonHeads.addAll(listRefLogObjects(ref, 0));
@ -838,7 +842,7 @@ public Collection<Pack> repack() throws IOException {
Pack heads = null;
if (!allHeadsAndTags.isEmpty()) {
heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
tagTargets, excluded);
refsToExcludeFromBitmap, tagTargets, excluded);
if (heads != null) {
ret.add(heads);
excluded.add(0, heads.getIndex());
@ -846,13 +850,13 @@ public Collection<Pack> repack() throws IOException {
}
if (!nonHeads.isEmpty()) {
Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
tagTargets, excluded);
PackWriter.NONE, tagTargets, excluded);
if (rest != null)
ret.add(rest);
}
if (!txnHeads.isEmpty()) {
Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
null, excluded);
PackWriter.NONE, null, excluded);
if (txn != null)
ret.add(txn);
}
@ -1017,10 +1021,7 @@ private void deleteTempPacksIdx() {
* @throws IOException
*/
private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
ReflogReader reflogReader = repo.getReflogReader(ref.getName());
if (reflogReader == null) {
return Collections.emptySet();
}
ReflogReader reflogReader = repo.getReflogReader(ref);
List<ReflogEntry> rlEntries = reflogReader
.getReverseEntries();
if (rlEntries == null || rlEntries.isEmpty())
@ -1124,6 +1125,7 @@ private Set<ObjectId> listNonHEADIndexObjects()
private Pack writePack(@NonNull Set<? extends ObjectId> want,
@NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
@NonNull Set<ObjectId> excludedRefsTips,
Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
throws IOException {
checkCancelled();
@ -1155,7 +1157,8 @@ private Pack writePack(@NonNull Set<? extends ObjectId> want,
if (excludeObjects != null)
for (ObjectIdSet idx : excludeObjects)
pw.excludeObjects(idx);
pw.preparePack(pm, want, have, PackWriter.NONE, tags);
pw.preparePack(pm, want, have, PackWriter.NONE,
union(tags, excludedRefsTips));
if (pw.getObjectCount() == 0)
return null;
checkCancelled();
@ -1268,6 +1271,15 @@ private Pack writePack(@NonNull Set<? extends ObjectId> want,
}
}
private Set<? extends ObjectId> union(Set<ObjectId> tags,
Set<ObjectId> excludedRefsHeadsTips) {
HashSet<ObjectId> unionSet = new HashSet<>(
tags.size() + excludedRefsHeadsTips.size());
unionSet.addAll(tags);
unionSet.addAll(excludedRefsHeadsTips);
return unionSet;
}
private void checkCancelled() throws CancelledException {
if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
throw new CancelledException(JGitText.get().operationCanceled);

3
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java
View File

@ -408,6 +408,9 @@ private CommitSelectionHelper captureOldAndNewCommits(RevWalk rw,
List<RevCommit> newWantsByNewest = new ArrayList<>(want.size());
Set<RevCommit> newWants = new HashSet<>(want.size());
for (AnyObjectId objectId : want) {
if(excludeFromBitmapSelection.contains(objectId)) {
continue;
}
RevObject ro = rw.peel(rw.parseAny(objectId));
if (!(ro instanceof RevCommit) || reuse.contains(ro)
|| excludeFromBitmapSelection.contains(ro)) {

6
org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchingProgressMonitor.java
View File

@ -176,7 +176,7 @@ void update(BatchingProgressMonitor pm, int completed) {
}
} else {
// Display once per second or when 1% is done.
int currPercent = lastWork * 100 / totalWork;
int currPercent = Math.round(lastWork * 100F / totalWork);
if (display) {
pm.onUpdate(taskName, lastWork, totalWork, currPercent);
output = true;
@ -201,8 +201,8 @@ void end(BatchingProgressMonitor pm) {
if (totalWork == UNKNOWN) {
pm.onEndTask(taskName, lastWork);
} else {
int pDone = lastWork * 100 / totalWork;
pm.onEndTask(taskName, lastWork, totalWork, pDone);
int currPercent = Math.round(lastWork * 100F / totalWork);
pm.onEndTask(taskName, lastWork, totalWork, currPercent);
}
}
if (timerFuture != null)

6
org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
View File

@ -720,6 +720,12 @@ public final class ConfigConstants {
*/
public static final String CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT = "bitmapexcessivebranchcount";
/**
* The "pack.bitmapExcludedRefsPrefixes" key
* @since 5.13.2
*/
public static final String CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES = "bitmapexcludedrefsprefixes";
/**
* The "pack.bitmapInactiveBranchAgeInDays" key
* @since 5.8

16
org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java
View File

@ -1694,6 +1694,22 @@ public void setGitwebDescription(@Nullable String description)
public abstract ReflogReader getReflogReader(String refName)
throws IOException;
/**
* Get the reflog reader. Subclasses should override this method and provide
* a more efficient implementation.
*
* @param ref
* a Ref
* @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref,
* or {@code null} if the ref does not exist.
* @throws IOException
* @since 5.13.2
*/
public @Nullable ReflogReader getReflogReader(@NonNull Ref ref)
throws IOException {
return getReflogReader(ref.getName());
}
/**
* Return the information stored in the file $GIT_DIR/MERGE_MSG. In this
* file operations triggering a merge will store a template for the commit

38
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
View File

@ -16,6 +16,7 @@
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_CONTIGUOUS_COMMIT_COUNT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BUILD_BITMAPS;
@ -225,6 +226,14 @@ public class PackConfig {
*/
public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90;
/**
* Default refs prefixes excluded from the calculation of pack bitmaps.
*
* @see #setBitmapExcludedRefsPrefixes(String[])
* @since 5.13.2
*/
public static final String[] DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES = new String[0];
/**
* Default max time to spend during the search for reuse phase. This
* optimization is disabled by default: {@value}
@ -285,6 +294,8 @@ public class PackConfig {
private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS;
private String[] bitmapExcludedRefsPrefixes = DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES;
private Duration searchForReuseTimeout = DEFAULT_SEARCH_FOR_REUSE_TIMEOUT;
private boolean cutDeltaChains;
@ -1144,6 +1155,27 @@ public void setBitmapInactiveBranchAgeInDays(int ageInDays) {
bitmapInactiveBranchAgeInDays = ageInDays;
}
/**
* Get the refs prefixes excluded from the Bitmap.
*
* @return the refs prefixes excluded from the Bitmap.
* @since 5.13.2
*/
public String[] getBitmapExcludedRefsPrefixes() {
return bitmapExcludedRefsPrefixes;
}
/**
* Set the refs prefixes excluded from the Bitmap.
*
* @param excludedRefsPrefixes
* the refs prefixes excluded from the Bitmap.
* @since 5.13.2
*/
public void setBitmapExcludedRefsPrefixes(String[] excludedRefsPrefixes) {
bitmapExcludedRefsPrefixes = excludedRefsPrefixes;
}
/**
* Set the max time to spend during the search for reuse phase.
*
@ -1220,6 +1252,12 @@ public void fromConfig(Config rc) {
setBitmapInactiveBranchAgeInDays(rc.getInt(CONFIG_PACK_SECTION,
CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS,
getBitmapInactiveBranchAgeInDays()));
String[] excludedRefsPrefixesArray = rc.getStringList(CONFIG_PACK_SECTION,
null,
CONFIG_KEY_BITMAP_EXCLUDED_REFS_PREFIXES);
if(excludedRefsPrefixesArray.length > 0) {
setBitmapExcludedRefsPrefixes(excludedRefsPrefixesArray);
}
setSearchForReuseTimeout(Duration.ofSeconds(rc.getTimeUnit(
CONFIG_PACK_SECTION, null,
CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT,

19
org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java
View File

@ -47,6 +47,7 @@
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.revwalk.ObjectWalk;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.util.StringUtils;
@ -387,11 +388,19 @@ private void updateFETCH_HEAD(FetchResult result) throws IOException {
private boolean askForIsComplete() throws TransportException {
try {
try (ObjectWalk ow = new ObjectWalk(transport.local)) {
for (ObjectId want : askFor.keySet())
ow.markStart(ow.parseAny(want));
for (Ref ref : localRefs().values())
ow.markUninteresting(ow.parseAny(ref.getObjectId()));
ow.checkConnectivity();
boolean hasCommitObject = false;
for (ObjectId want : askFor.keySet()) {
RevObject obj = ow.parseAny(want);
ow.markStart(obj);
hasCommitObject |= obj.getType() == Constants.OBJ_COMMIT;
}
// Checking connectivity makes sense on commits only
if (hasCommitObject) {
for (Ref ref : localRefs().values()) {
ow.markUninteresting(ow.parseAny(ref.getObjectId()));
}
ow.checkConnectivity();
}
}
return transport.getDepth() == null; // if depth is set we need to request objects that are already available
} catch (MissingObjectException e) {