motiejus
/
jgit
1
Fork 0
Code Releases Activity

DFS block cache: add additional stats to DfsReaderIoStats 

New stats are populated in DfsPackFile to provide details about loading
pack, bitmap and reverse indexes.

Signed-off-by: Alina Djamankulova <adjama@google.com>
Change-Id: Ib6e8384ecc31821261e5099d0768c0b1227e8364
This commit is contained in:
Alina Djamankulova 2021-08-17 09:56:06 -07:00 committed by Matthias Sohn
parent cf9baedb5d
commit 8173c16785
2 changed files with 121 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
View File

@ -25,6 +25,7 @@
import java.nio.channels.Channels;
import java.text.MessageFormat;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
@ -164,10 +165,17 @@ private PackIndex idx(DfsReader ctx) throws IOException {
try {
DfsStreamKey idxKey = desc.getStreamKey(INDEX);
AtomicBoolean cacheHit = new AtomicBoolean(true);
DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
idxKey,
REF_POSITION,
() -> loadPackIndex(ctx, idxKey));
() -> {
cacheHit.set(false);
return loadPackIndex(ctx, idxKey);
});
if (cacheHit.get()) {
ctx.stats.idxCacheHit++;
}
PackIndex idx = idxref.get();
if (index == null && idx != null) {
index = idx;
@ -210,10 +218,17 @@ public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
}
DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
AtomicBoolean cacheHit = new AtomicBoolean(true);
DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
bitmapKey,
REF_POSITION,
() -> loadBitmapIndex(ctx, bitmapKey));
() -> {
cacheHit.set(false);
return loadBitmapIndex(ctx, bitmapKey);
});
if (cacheHit.get()) {
ctx.stats.bitmapCacheHit++;
}
PackBitmapIndex bmidx = idxref.get();
if (bitmapIndex == null && bmidx != null) {
bitmapIndex = bmidx;
@ -235,10 +250,17 @@ PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
PackIndex idx = idx(ctx);
DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
desc.getStreamKey(INDEX));
AtomicBoolean cacheHit = new AtomicBoolean(true);
DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
revKey,
REF_POSITION,
() -> loadReverseIdx(revKey, idx));
() -> {
cacheHit.set(false);
return loadReverseIdx(ctx, revKey, idx);
});
if (cacheHit.get()) {
ctx.stats.ridxCacheHit++;
}
PackReverseIndex revidx = revref.get();
if (reverseIndex == null && revidx != null) {
reverseIndex = revidx;
@ -1041,9 +1063,12 @@ private DfsBlockCache.Ref<PackIndex> loadPackIndex(
}
private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
DfsStreamKey revKey, PackIndex idx) {
DfsReader ctx, DfsStreamKey revKey, PackIndex idx) {
ctx.stats.readReverseIdx++;
long start = System.nanoTime();
PackReverseIndex revidx = new PackReverseIndex(idx);
reverseIndex = revidx;
ctx.stats.readReverseIdxMicros += elapsedMicros(start);
return new DfsBlockCache.Ref<>(
revKey,
REF_POSITION,
@ -1072,8 +1097,8 @@ private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(DfsReader ctx,
() -> getReverseIdx(ctx));
} finally {
size = rc.position();
ctx.stats.readIdxBytes += size;
ctx.stats.readIdxMicros += elapsedMicros(start);
ctx.stats.readBitmapIdxBytes += size;
ctx.stats.readBitmapIdxMicros += elapsedMicros(start);
}
bitmapIndex = bmidx;
return new DfsBlockCache.Ref<>(

96
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java
View File

@ -19,18 +19,39 @@ public static class Accumulator {
/** Number of times the reader explicitly called scanPacks. */
long scanPacks;
/** Total number of cache hits for pack indexes. */
long idxCacheHit;
/** Total number of cache hits for reverse indexes. */
long ridxCacheHit;
/** Total number of cache hits for bitmap indexes. */
long bitmapCacheHit;
/** Total number of complete pack indexes read into memory. */
long readIdx;
/** Total number of complete bitmap indexes read into memory. */
long readBitmap;
/** Total number of bytes read from indexes. */
/** Total number of reverse indexes added into memory. */
long readReverseIdx;
/** Total number of bytes read from pack indexes. */
long readIdxBytes;
/** Total microseconds spent reading pack or bitmap indexes. */
/** Total microseconds spent reading pack indexes. */
long readIdxMicros;
/** Total microseconds spent creating reverse indexes. */
long readReverseIdxMicros;
/** Total number of bytes read from bitmap indexes. */
long readBitmapIdxBytes;
/** Total microseconds spent reading bitmap indexes. */
long readBitmapIdxMicros;
/** Total number of block cache hits. */
long blockCacheHit;
@ -74,6 +95,33 @@ public long getScanPacks() {
return stats.scanPacks;
}
/**
* Get total number of pack index cache hits.
*
* @return total number of pack index cache hits.
*/
public long getPackIndexCacheHits() {
return stats.idxCacheHit;
}
/**
* Get total number of reverse index cache hits.
*
* @return total number of reverse index cache hits.
*/
public long getReverseIndexCacheHits() {
return stats.ridxCacheHit;
}
/**
* Get total number of bitmap index cache hits.
*
* @return total number of bitmap index cache hits.
*/
public long getBitmapIndexCacheHits() {
return stats.bitmapCacheHit;
}
/**
* Get total number of complete pack indexes read into memory.
*
@ -83,6 +131,15 @@ public long getReadPackIndexCount() {
return stats.readIdx;
}
/**
* Get total number of times the reverse index was computed.
*
* @return total number of reverse index was computed.
*/
public long getReadReverseIndexCount() {
return stats.readReverseIdx;
}
/**
* Get total number of complete bitmap indexes read into memory.
*
@ -93,23 +150,50 @@ public long getReadBitmapIndexCount() {
}
/**
* Get total number of bytes read from indexes.
* Get total number of bytes read from pack indexes.
*
* @return total number of bytes read from indexes.
* @return total number of bytes read from pack indexes.
*/
public long getReadIndexBytes() {
return stats.readIdxBytes;
}
/**
* Get total microseconds spent reading pack or bitmap indexes.
* Get total microseconds spent reading pack indexes.
*
* @return total microseconds spent reading pack or bitmap indexes.
* @return total microseconds spent reading pack indexes.
*/
public long getReadIndexMicros() {
return stats.readIdxMicros;
}
/**
* Get total microseconds spent creating reverse indexes.
*
* @return total microseconds spent creating reverse indexes.
*/
public long getReadReverseIndexMicros() {
return stats.readReverseIdxMicros;
}
/**
* Get total number of bytes read from bitmap indexes.
*
* @return total number of bytes read from bitmap indexes.
*/
public long getReadBitmapIndexBytes() {
return stats.readBitmapIdxBytes;
}
/**
* Get total microseconds spent reading bitmap indexes.
*
* @return total microseconds spent reading bitmap indexes.
*/
public long getReadBitmapIndexMicros() {
return stats.readBitmapIdxMicros;
}
/**
* Get total number of block cache hits.
*