Merge branch 'stable-5.6' into stable-5.7
# By Matthias Sohn (3) and Han-Wen Nienhuys (1) * stable-5.6: Update API problem filter Prepare 5.6.2-SNAPSHOT builds JGit v5.6.1.202002131546-r Simplify ReftableCompactor Change-Id: I16ed174f9fc662934c3ebaea85a60690efbed1c6 Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
This commit is contained in:
commit
7a940542ee
|
@ -19,7 +19,9 @@
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.eclipse.jgit.internal.storage.io.BlockSource;
|
import org.eclipse.jgit.internal.storage.io.BlockSource;
|
||||||
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter.Stats;
|
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter.Stats;
|
||||||
|
@ -63,7 +65,9 @@ public void oneTable() throws IOException {
|
||||||
ReftableCompactor compactor;
|
ReftableCompactor compactor;
|
||||||
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
|
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
|
||||||
compactor = new ReftableCompactor(outBuf);
|
compactor = new ReftableCompactor(outBuf);
|
||||||
compactor.tryAddFirst(read(inTab));
|
List<ReftableReader> readers = new ArrayList<>();
|
||||||
|
readers.add(read(inTab));
|
||||||
|
compactor.addAll(readers);
|
||||||
compactor.compact();
|
compactor.compact();
|
||||||
outTab = outBuf.toByteArray();
|
outTab = outBuf.toByteArray();
|
||||||
}
|
}
|
||||||
|
|
|
@ -452,10 +452,6 @@ private File compactLocked(int first, int last) throws IOException {
|
||||||
try (FileOutputStream fos = new FileOutputStream(tmpTable)) {
|
try (FileOutputStream fos = new FileOutputStream(tmpTable)) {
|
||||||
ReftableCompactor c = new ReftableCompactor(fos)
|
ReftableCompactor c = new ReftableCompactor(fos)
|
||||||
.setConfig(reftableConfig())
|
.setConfig(reftableConfig())
|
||||||
.setMinUpdateIndex(
|
|
||||||
stack.get(first).reftableReader.minUpdateIndex())
|
|
||||||
.setMaxUpdateIndex(
|
|
||||||
stack.get(last).reftableReader.maxUpdateIndex())
|
|
||||||
.setIncludeDeletes(first > 0);
|
.setIncludeDeletes(first > 0);
|
||||||
|
|
||||||
List<ReftableReader> compactMe = new ArrayList<>();
|
List<ReftableReader> compactMe = new ArrayList<>();
|
||||||
|
|
|
@ -65,6 +65,15 @@ public long maxUpdateIndex() throws IOException {
|
||||||
: 0;
|
: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public long minUpdateIndex() throws IOException {
|
||||||
|
return tables.length > 0 ? tables[0].minUpdateIndex()
|
||||||
|
: 0;
|
||||||
|
}
|
||||||
|
|
||||||
/** {@inheritDoc} */
|
/** {@inheritDoc} */
|
||||||
@Override
|
@Override
|
||||||
public boolean hasObjectMap() throws IOException {
|
public boolean hasObjectMap() throws IOException {
|
||||||
|
|
|
@ -65,19 +65,28 @@ public void setIncludeDeletes(boolean deletes) {
|
||||||
includeDeletes = deletes;
|
includeDeletes = deletes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the maximum update index for log entries that appear in this
|
* Get the maximum update index for ref entries that appear in this
|
||||||
* reftable.
|
* reftable.
|
||||||
*
|
*
|
||||||
* @return the maximum update index for log entries that appear in this
|
* @return the maximum update index for ref entries that appear in this
|
||||||
* reftable. This should be 1 higher than the prior reftable's
|
* reftable.
|
||||||
* {@code maxUpdateIndex} if this table is used in a stack.
|
|
||||||
* @throws java.io.IOException
|
* @throws java.io.IOException
|
||||||
* file cannot be read.
|
* file cannot be read.
|
||||||
*/
|
*/
|
||||||
public abstract long maxUpdateIndex() throws IOException;
|
public abstract long maxUpdateIndex() throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the minimum update index for ref entries that appear in this
|
||||||
|
* reftable.
|
||||||
|
*
|
||||||
|
* @return the minimum update index for ref entries that appear in this
|
||||||
|
* reftable.
|
||||||
|
* @throws java.io.IOException
|
||||||
|
* file cannot be read.
|
||||||
|
*/
|
||||||
|
public abstract long minUpdateIndex() throws IOException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Seek to the first reference, to iterate in order.
|
* Seek to the first reference, to iterate in order.
|
||||||
*
|
*
|
||||||
|
|
|
@ -28,22 +28,20 @@
|
||||||
* to shadow any lower reftable that may have the reference present.
|
* to shadow any lower reftable that may have the reference present.
|
||||||
* <p>
|
* <p>
|
||||||
* By default all log entries within the range defined by
|
* By default all log entries within the range defined by
|
||||||
* {@link #setMinUpdateIndex(long)} and {@link #setMaxUpdateIndex(long)} are
|
* {@link #setReflogExpireMinUpdateIndex(long)} and {@link #setReflogExpireMaxUpdateIndex(long)} are
|
||||||
* copied, even if no references in the output file match the log records.
|
* copied, even if no references in the output file match the log records.
|
||||||
* Callers may truncate the log to a more recent time horizon with
|
* Callers may truncate the log to a more recent time horizon with
|
||||||
* {@link #setOldestReflogTimeMillis(long)}, or disable the log altogether with
|
* {@link #setReflogExpireOldestReflogTimeMillis(long)}, or disable the log altogether with
|
||||||
* {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}.
|
* {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}.
|
||||||
*/
|
*/
|
||||||
public class ReftableCompactor {
|
public class ReftableCompactor {
|
||||||
private final ReftableWriter writer;
|
private final ReftableWriter writer;
|
||||||
private final ArrayDeque<ReftableReader> tables = new ArrayDeque<>();
|
private final ArrayDeque<ReftableReader> tables = new ArrayDeque<>();
|
||||||
|
|
||||||
private long compactBytesLimit;
|
|
||||||
private long bytesToCompact;
|
|
||||||
private boolean includeDeletes;
|
private boolean includeDeletes;
|
||||||
private long minUpdateIndex = -1;
|
private long reflogExpireMinUpdateIndex = 0;
|
||||||
private long maxUpdateIndex;
|
private long reflogExpireMaxUpdateIndex = Long.MAX_VALUE;
|
||||||
private long oldestReflogTimeMillis;
|
private long reflogExpireOldestReflogTimeMillis;
|
||||||
private Stats stats;
|
private Stats stats;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -69,18 +67,6 @@ public ReftableCompactor setConfig(ReftableConfig cfg) {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Set limit on number of bytes from source tables to compact.
|
|
||||||
*
|
|
||||||
* @param bytes
|
|
||||||
* limit on number of bytes from source tables to compact.
|
|
||||||
* @return {@code this}
|
|
||||||
*/
|
|
||||||
public ReftableCompactor setCompactBytesLimit(long bytes) {
|
|
||||||
compactBytesLimit = bytes;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether to include deletions in the output, which may be necessary for
|
* Whether to include deletions in the output, which may be necessary for
|
||||||
* partial compaction.
|
* partial compaction.
|
||||||
|
@ -106,8 +92,8 @@ public ReftableCompactor setIncludeDeletes(boolean deletes) {
|
||||||
* in a stack.
|
* in a stack.
|
||||||
* @return {@code this}
|
* @return {@code this}
|
||||||
*/
|
*/
|
||||||
public ReftableCompactor setMinUpdateIndex(long min) {
|
public ReftableCompactor setReflogExpireMinUpdateIndex(long min) {
|
||||||
minUpdateIndex = min;
|
reflogExpireMinUpdateIndex = min;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -122,8 +108,8 @@ public ReftableCompactor setMinUpdateIndex(long min) {
|
||||||
* used in a stack.
|
* used in a stack.
|
||||||
* @return {@code this}
|
* @return {@code this}
|
||||||
*/
|
*/
|
||||||
public ReftableCompactor setMaxUpdateIndex(long max) {
|
public ReftableCompactor setReflogExpireMaxUpdateIndex(long max) {
|
||||||
maxUpdateIndex = max;
|
reflogExpireMaxUpdateIndex = max;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -137,16 +123,13 @@ public ReftableCompactor setMaxUpdateIndex(long max) {
|
||||||
* Specified in Java standard milliseconds since the epoch.
|
* Specified in Java standard milliseconds since the epoch.
|
||||||
* @return {@code this}
|
* @return {@code this}
|
||||||
*/
|
*/
|
||||||
public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) {
|
public ReftableCompactor setReflogExpireOldestReflogTimeMillis(long timeMillis) {
|
||||||
oldestReflogTimeMillis = timeMillis;
|
reflogExpireOldestReflogTimeMillis = timeMillis;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Add all of the tables, in the specified order.
|
* Add all of the tables, in the specified order.
|
||||||
* <p>
|
|
||||||
* Unconditionally adds all tables, ignoring the
|
|
||||||
* {@link #setCompactBytesLimit(long)}.
|
|
||||||
*
|
*
|
||||||
* @param readers
|
* @param readers
|
||||||
* tables to compact. Tables should be ordered oldest first/most
|
* tables to compact. Tables should be ordered oldest first/most
|
||||||
|
@ -158,46 +141,9 @@ public ReftableCompactor setOldestReflogTimeMillis(long timeMillis) {
|
||||||
public void addAll(List<ReftableReader> readers) throws IOException {
|
public void addAll(List<ReftableReader> readers) throws IOException {
|
||||||
for (ReftableReader r : readers) {
|
for (ReftableReader r : readers) {
|
||||||
tables.add(r);
|
tables.add(r);
|
||||||
adjustUpdateIndexes(r);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Try to add this reader at the bottom of the stack.
|
|
||||||
* <p>
|
|
||||||
* A reader may be rejected by returning {@code false} if the compactor is
|
|
||||||
* already rewriting its {@link #setCompactBytesLimit(long)}. When this
|
|
||||||
* happens the caller should stop trying to add tables, and execute the
|
|
||||||
* compaction.
|
|
||||||
*
|
|
||||||
* @param reader
|
|
||||||
* the reader to insert at the bottom of the stack. Caller is
|
|
||||||
* responsible for closing the reader.
|
|
||||||
* @return {@code true} if the compactor accepted this table; {@code false}
|
|
||||||
* if the compactor has reached its limit.
|
|
||||||
* @throws java.io.IOException
|
|
||||||
* if size of {@code reader}, or its update indexes cannot be read.
|
|
||||||
*/
|
|
||||||
public boolean tryAddFirst(ReftableReader reader) throws IOException {
|
|
||||||
long sz = reader.size();
|
|
||||||
if (compactBytesLimit > 0 && bytesToCompact + sz > compactBytesLimit) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bytesToCompact += sz;
|
|
||||||
adjustUpdateIndexes(reader);
|
|
||||||
tables.addFirst(reader);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void adjustUpdateIndexes(ReftableReader reader) throws IOException {
|
|
||||||
if (minUpdateIndex == -1) {
|
|
||||||
minUpdateIndex = reader.minUpdateIndex();
|
|
||||||
} else {
|
|
||||||
minUpdateIndex = Math.min(minUpdateIndex, reader.minUpdateIndex());
|
|
||||||
}
|
|
||||||
maxUpdateIndex = Math.max(maxUpdateIndex, reader.maxUpdateIndex());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write a compaction to {@code out}.
|
* Write a compaction to {@code out}.
|
||||||
*
|
*
|
||||||
|
@ -208,8 +154,9 @@ public void compact() throws IOException {
|
||||||
MergedReftable mr = new MergedReftable(new ArrayList<>(tables));
|
MergedReftable mr = new MergedReftable(new ArrayList<>(tables));
|
||||||
mr.setIncludeDeletes(includeDeletes);
|
mr.setIncludeDeletes(includeDeletes);
|
||||||
|
|
||||||
writer.setMinUpdateIndex(Math.max(minUpdateIndex, 0));
|
writer.setMaxUpdateIndex(mr.maxUpdateIndex());
|
||||||
writer.setMaxUpdateIndex(maxUpdateIndex);
|
writer.setMinUpdateIndex(mr.minUpdateIndex());
|
||||||
|
|
||||||
writer.begin();
|
writer.begin();
|
||||||
mergeRefs(mr);
|
mergeRefs(mr);
|
||||||
mergeLogs(mr);
|
mergeLogs(mr);
|
||||||
|
@ -235,16 +182,14 @@ private void mergeRefs(MergedReftable mr) throws IOException {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLogs(MergedReftable mr) throws IOException {
|
private void mergeLogs(MergedReftable mr) throws IOException {
|
||||||
if (oldestReflogTimeMillis == Long.MAX_VALUE) {
|
if (reflogExpireOldestReflogTimeMillis == Long.MAX_VALUE) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
try (LogCursor lc = mr.allLogs()) {
|
try (LogCursor lc = mr.allLogs()) {
|
||||||
while (lc.next()) {
|
while (lc.next()) {
|
||||||
long updateIndex = lc.getUpdateIndex();
|
long updateIndex = lc.getUpdateIndex();
|
||||||
if (updateIndex < minUpdateIndex
|
if (updateIndex > reflogExpireMaxUpdateIndex || updateIndex < reflogExpireMinUpdateIndex) {
|
||||||
|| updateIndex > maxUpdateIndex) {
|
|
||||||
// Cannot merge log records outside the header's range.
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -258,7 +203,7 @@ private void mergeLogs(MergedReftable mr) throws IOException {
|
||||||
}
|
}
|
||||||
|
|
||||||
PersonIdent who = log.getWho();
|
PersonIdent who = log.getWho();
|
||||||
if (who.getWhen().getTime() >= oldestReflogTimeMillis) {
|
if (who.getWhen().getTime() >= reflogExpireOldestReflogTimeMillis) {
|
||||||
writer.writeLog(
|
writer.writeLog(
|
||||||
refName,
|
refName,
|
||||||
updateIndex,
|
updateIndex,
|
||||||
|
|
|
@ -106,15 +106,9 @@ public boolean hasObjectMap() throws IOException {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the minimum update index for log entries that appear in this
|
* {@inheritDoc}
|
||||||
* reftable.
|
|
||||||
*
|
|
||||||
* @return the minimum update index for log entries that appear in this
|
|
||||||
* reftable. This should be 1 higher than the prior reftable's
|
|
||||||
* {@code maxUpdateIndex} if this table is used in a stack.
|
|
||||||
* @throws java.io.IOException
|
|
||||||
* file cannot be read.
|
|
||||||
*/
|
*/
|
||||||
|
@Override
|
||||||
public long minUpdateIndex() throws IOException {
|
public long minUpdateIndex() throws IOException {
|
||||||
if (blockSize == -1) {
|
if (blockSize == -1) {
|
||||||
readFileHeader();
|
readFileHeader();
|
||||||
|
|
Loading…
Reference in New Issue