Merge branch 'stable-5.6' into stable-5.7

# By Matthias Sohn (3) and Han-Wen Nienhuys (1)
* stable-5.6:
  Update API problem filter
  Prepare 5.6.2-SNAPSHOT builds
  JGit v5.6.1.202002131546-r
  Simplify ReftableCompactor

Change-Id: I16ed174f9fc662934c3ebaea85a60690efbed1c6
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
This commit is contained in:
Matthias Sohn 2020-02-19 01:44:18 +01:00
commit 7a940542ee
6 changed files with 47 additions and 90 deletions

View File

@ -19,7 +19,9 @@
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import org.eclipse.jgit.internal.storage.io.BlockSource; import org.eclipse.jgit.internal.storage.io.BlockSource;
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter.Stats; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter.Stats;
@ -63,7 +65,9 @@ public void oneTable() throws IOException {
ReftableCompactor compactor; ReftableCompactor compactor;
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
compactor = new ReftableCompactor(outBuf); compactor = new ReftableCompactor(outBuf);
compactor.tryAddFirst(read(inTab)); List<ReftableReader> readers = new ArrayList<>();
readers.add(read(inTab));
compactor.addAll(readers);
compactor.compact(); compactor.compact();
outTab = outBuf.toByteArray(); outTab = outBuf.toByteArray();
} }

View File

@ -452,10 +452,6 @@ private File compactLocked(int first, int last) throws IOException {
try (FileOutputStream fos = new FileOutputStream(tmpTable)) { try (FileOutputStream fos = new FileOutputStream(tmpTable)) {
ReftableCompactor c = new ReftableCompactor(fos) ReftableCompactor c = new ReftableCompactor(fos)
.setConfig(reftableConfig()) .setConfig(reftableConfig())
.setMinUpdateIndex(
stack.get(first).reftableReader.minUpdateIndex())
.setMaxUpdateIndex(
stack.get(last).reftableReader.maxUpdateIndex())
.setIncludeDeletes(first > 0); .setIncludeDeletes(first > 0);
List<ReftableReader> compactMe = new ArrayList<>(); List<ReftableReader> compactMe = new ArrayList<>();

View File

@ -65,6 +65,15 @@ public long maxUpdateIndex() throws IOException {
: 0; : 0;
} }
/**
* {@inheritDoc}
*/
@Override
public long minUpdateIndex() throws IOException {
return tables.length > 0 ? tables[0].minUpdateIndex()
: 0;
}
/** {@inheritDoc} */ /** {@inheritDoc} */
@Override @Override
public boolean hasObjectMap() throws IOException { public boolean hasObjectMap() throws IOException {

View File

@ -65,19 +65,28 @@ public void setIncludeDeletes(boolean deletes) {
includeDeletes = deletes; includeDeletes = deletes;
} }
/** /**
* Get the maximum update index for log entries that appear in this * Get the maximum update index for ref entries that appear in this
* reftable. * reftable.
* *
* @return the maximum update index for log entries that appear in this * @return the maximum update index for ref entries that appear in this
* reftable. This should be 1 higher than the prior reftable's * reftable.
* {@code maxUpdateIndex} if this table is used in a stack.
* @throws java.io.IOException * @throws java.io.IOException
* file cannot be read. * file cannot be read.
*/ */
public abstract long maxUpdateIndex() throws IOException; public abstract long maxUpdateIndex() throws IOException;
/**
* Get the minimum update index for ref entries that appear in this
* reftable.
*
* @return the minimum update index for ref entries that appear in this
* reftable.
* @throws java.io.IOException
* file cannot be read.
*/
public abstract long minUpdateIndex() throws IOException;
/** /**
* Seek to the first reference, to iterate in order. * Seek to the first reference, to iterate in order.
* *

View File

@ -28,22 +28,20 @@
* to shadow any lower reftable that may have the reference present. * to shadow any lower reftable that may have the reference present.
* <p> * <p>
* By default all log entries within the range defined by * By default all log entries within the range defined by
* {@link #setMinUpdateIndex(long)} and {@link #setMaxUpdateIndex(long)} are * {@link #setReflogExpireMinUpdateIndex(long)} and {@link #setReflogExpireMaxUpdateIndex(long)} are
* copied, even if no references in the output file match the log records. * copied, even if no references in the output file match the log records.
* Callers may truncate the log to a more recent time horizon with * Callers may truncate the log to a more recent time horizon with
* {@link #setOldestReflogTimeMillis(long)}, or disable the log altogether with * {@link #setReflogExpireOldestReflogTimeMillis(long)}, or disable the log altogether with
* {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}. * {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}.
*/ */
public class ReftableCompactor { public class ReftableCompactor {
private final ReftableWriter writer; private final ReftableWriter writer;
private final ArrayDeque<ReftableReader> tables = new ArrayDeque<>(); private final ArrayDeque<ReftableReader> tables = new ArrayDeque<>();
private long compactBytesLimit;
private long bytesToCompact;
private boolean includeDeletes; private boolean includeDeletes;
private long minUpdateIndex = -1; private long reflogExpireMinUpdateIndex = 0;
private long maxUpdateIndex; private long reflogExpireMaxUpdateIndex = Long.MAX_VALUE;
private long oldestReflogTimeMillis; private long reflogExpireOldestReflogTimeMillis;
private Stats stats; private Stats stats;
/** /**
@ -69,18 +67,6 @@ public ReftableCompactor setConfig(ReftableConfig cfg) {
return this; return this;
} }
/**
* Set limit on number of bytes from source tables to compact.
*
* @param bytes
* limit on number of bytes from source tables to compact.
* @return {@code this}
*/
public ReftableCompactor setCompactBytesLimit(long bytes) {
compactBytesLimit = bytes;
return this;
}
/** /**
* Whether to include deletions in the output, which may be necessary for * Whether to include deletions in the output, which may be necessary for
* partial compaction. * partial compaction.
@ -106,8 +92,8 @@ public ReftableCompactor setIncludeDeletes(boolean deletes) {
* in a stack. * in a stack.
* @return {@code this} * @return {@code this}
*/ */
public ReftableCompactor setMinUpdateIndex(long min) { public ReftableCompactor setReflogExpireMinUpdateIndex(long min) {
minUpdateIndex = min; reflogExpireMinUpdateIndex = min;
return this; return this;
} }
@ -122,8 +108,8 @@ public ReftableCompactor setMinUpdateIndex(long min) {
* used in a stack. * used in a stack.
* @return {@code this} * @return {@code this}
*/ */
public ReftableCompactor setMaxUpdateIndex(long max) { public ReftableCompactor setReflogExpireMaxUpdateIndex(long max) {
maxUpdateIndex = max; reflogExpireMaxUpdateIndex = max;
return this; return this;
} }
@ -137,16 +123,13 @@ public ReftableCompactor setMaxUpdateIndex(long max) {
* Specified in Java standard milliseconds since the epoch. * Specified in Java standard milliseconds since the epoch.
* @return {@code this} * @return {@code this}
*/ */
public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) { public ReftableCompactor setReflogExpireOldestReflogTimeMillis(long timeMillis) {
oldestReflogTimeMillis = timeMillis; reflogExpireOldestReflogTimeMillis = timeMillis;
return this; return this;
} }
/** /**
* Add all of the tables, in the specified order. * Add all of the tables, in the specified order.
* <p>
* Unconditionally adds all tables, ignoring the
* {@link #setCompactBytesLimit(long)}.
* *
* @param readers * @param readers
* tables to compact. Tables should be ordered oldest first/most * tables to compact. Tables should be ordered oldest first/most
@ -158,46 +141,9 @@ public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) {
public void addAll(List<ReftableReader> readers) throws IOException { public void addAll(List<ReftableReader> readers) throws IOException {
for (ReftableReader r : readers) { for (ReftableReader r : readers) {
tables.add(r); tables.add(r);
adjustUpdateIndexes(r);
} }
} }
/**
* Try to add this reader at the bottom of the stack.
* <p>
* A reader may be rejected by returning {@code false} if the compactor is
* already rewriting its {@link #setCompactBytesLimit(long)}. When this
* happens the caller should stop trying to add tables, and execute the
* compaction.
*
* @param reader
* the reader to insert at the bottom of the stack. Caller is
* responsible for closing the reader.
* @return {@code true} if the compactor accepted this table; {@code false}
* if the compactor has reached its limit.
* @throws java.io.IOException
* if size of {@code reader}, or its update indexes cannot be read.
*/
public boolean tryAddFirst(ReftableReader reader) throws IOException {
long sz = reader.size();
if (compactBytesLimit > 0 && bytesToCompact + sz > compactBytesLimit) {
return false;
}
bytesToCompact += sz;
adjustUpdateIndexes(reader);
tables.addFirst(reader);
return true;
}
private void adjustUpdateIndexes(ReftableReader reader) throws IOException {
if (minUpdateIndex == -1) {
minUpdateIndex = reader.minUpdateIndex();
} else {
minUpdateIndex = Math.min(minUpdateIndex, reader.minUpdateIndex());
}
maxUpdateIndex = Math.max(maxUpdateIndex, reader.maxUpdateIndex());
}
/** /**
* Write a compaction to {@code out}. * Write a compaction to {@code out}.
* *
@ -208,8 +154,9 @@ public void compact() throws IOException {
MergedReftable mr = new MergedReftable(new ArrayList<>(tables)); MergedReftable mr = new MergedReftable(new ArrayList<>(tables));
mr.setIncludeDeletes(includeDeletes); mr.setIncludeDeletes(includeDeletes);
writer.setMinUpdateIndex(Math.max(minUpdateIndex, 0)); writer.setMaxUpdateIndex(mr.maxUpdateIndex());
writer.setMaxUpdateIndex(maxUpdateIndex); writer.setMinUpdateIndex(mr.minUpdateIndex());
writer.begin(); writer.begin();
mergeRefs(mr); mergeRefs(mr);
mergeLogs(mr); mergeLogs(mr);
@ -235,16 +182,14 @@ private void mergeRefs(MergedReftable mr) throws IOException {
} }
private void mergeLogs(MergedReftable mr) throws IOException { private void mergeLogs(MergedReftable mr) throws IOException {
if (oldestReflogTimeMillis == Long.MAX_VALUE) { if (reflogExpireOldestReflogTimeMillis == Long.MAX_VALUE) {
return; return;
} }
try (LogCursor lc = mr.allLogs()) { try (LogCursor lc = mr.allLogs()) {
while (lc.next()) { while (lc.next()) {
long updateIndex = lc.getUpdateIndex(); long updateIndex = lc.getUpdateIndex();
if (updateIndex < minUpdateIndex if (updateIndex > reflogExpireMaxUpdateIndex || updateIndex < reflogExpireMinUpdateIndex) {
|| updateIndex > maxUpdateIndex) {
// Cannot merge log records outside the header's range.
continue; continue;
} }
@ -258,7 +203,7 @@ private void mergeLogs(MergedReftable mr) throws IOException {
} }
PersonIdent who = log.getWho(); PersonIdent who = log.getWho();
if (who.getWhen().getTime() >= oldestReflogTimeMillis) { if (who.getWhen().getTime() >= reflogExpireOldestReflogTimeMillis) {
writer.writeLog( writer.writeLog(
refName, refName,
updateIndex, updateIndex,

View File

@ -106,15 +106,9 @@ public boolean hasObjectMap() throws IOException {
} }
/** /**
* Get the minimum update index for log entries that appear in this * {@inheritDoc}
* reftable.
*
* @return the minimum update index for log entries that appear in this
* reftable. This should be 1 higher than the prior reftable's
* {@code maxUpdateIndex} if this table is used in a stack.
* @throws java.io.IOException
* file cannot be read.
*/ */
@Override
public long minUpdateIndex() throws IOException { public long minUpdateIndex() throws IOException {
if (blockSize == -1) { if (blockSize == -1) {
readFileHeader(); readFileHeader();