DFS block cache: add additional stats to DfsReaderIoStats
New stats are populated in DfsPackFile to provide details about loading pack, bitmap and reverse indexes. Signed-off-by: Alina Djamankulova <adjama@google.com> Change-Id: Ib6e8384ecc31821261e5099d0768c0b1227e8364
This commit is contained in:
parent
cf9baedb5d
commit
8173c16785
|
@ -25,6 +25,7 @@
|
|||
import java.nio.channels.Channels;
|
||||
import java.text.MessageFormat;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.zip.CRC32;
|
||||
import java.util.zip.DataFormatException;
|
||||
import java.util.zip.Inflater;
|
||||
|
@ -164,10 +165,17 @@ private PackIndex idx(DfsReader ctx) throws IOException {
|
|||
|
||||
try {
|
||||
DfsStreamKey idxKey = desc.getStreamKey(INDEX);
|
||||
AtomicBoolean cacheHit = new AtomicBoolean(true);
|
||||
DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
|
||||
idxKey,
|
||||
REF_POSITION,
|
||||
() -> loadPackIndex(ctx, idxKey));
|
||||
() -> {
|
||||
cacheHit.set(false);
|
||||
return loadPackIndex(ctx, idxKey);
|
||||
});
|
||||
if (cacheHit.get()) {
|
||||
ctx.stats.idxCacheHit++;
|
||||
}
|
||||
PackIndex idx = idxref.get();
|
||||
if (index == null && idx != null) {
|
||||
index = idx;
|
||||
|
@ -210,10 +218,17 @@ public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
|
|||
}
|
||||
|
||||
DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
|
||||
AtomicBoolean cacheHit = new AtomicBoolean(true);
|
||||
DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
|
||||
bitmapKey,
|
||||
REF_POSITION,
|
||||
() -> loadBitmapIndex(ctx, bitmapKey));
|
||||
() -> {
|
||||
cacheHit.set(false);
|
||||
return loadBitmapIndex(ctx, bitmapKey);
|
||||
});
|
||||
if (cacheHit.get()) {
|
||||
ctx.stats.bitmapCacheHit++;
|
||||
}
|
||||
PackBitmapIndex bmidx = idxref.get();
|
||||
if (bitmapIndex == null && bmidx != null) {
|
||||
bitmapIndex = bmidx;
|
||||
|
@ -235,10 +250,17 @@ PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
|
|||
PackIndex idx = idx(ctx);
|
||||
DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
|
||||
desc.getStreamKey(INDEX));
|
||||
AtomicBoolean cacheHit = new AtomicBoolean(true);
|
||||
DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
|
||||
revKey,
|
||||
REF_POSITION,
|
||||
() -> loadReverseIdx(revKey, idx));
|
||||
() -> {
|
||||
cacheHit.set(false);
|
||||
return loadReverseIdx(ctx, revKey, idx);
|
||||
});
|
||||
if (cacheHit.get()) {
|
||||
ctx.stats.ridxCacheHit++;
|
||||
}
|
||||
PackReverseIndex revidx = revref.get();
|
||||
if (reverseIndex == null && revidx != null) {
|
||||
reverseIndex = revidx;
|
||||
|
@ -1041,9 +1063,12 @@ private DfsBlockCache.Ref<PackIndex> loadPackIndex(
|
|||
}
|
||||
|
||||
private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
|
||||
DfsStreamKey revKey, PackIndex idx) {
|
||||
DfsReader ctx, DfsStreamKey revKey, PackIndex idx) {
|
||||
ctx.stats.readReverseIdx++;
|
||||
long start = System.nanoTime();
|
||||
PackReverseIndex revidx = new PackReverseIndex(idx);
|
||||
reverseIndex = revidx;
|
||||
ctx.stats.readReverseIdxMicros += elapsedMicros(start);
|
||||
return new DfsBlockCache.Ref<>(
|
||||
revKey,
|
||||
REF_POSITION,
|
||||
|
@ -1072,8 +1097,8 @@ private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(DfsReader ctx,
|
|||
() -> getReverseIdx(ctx));
|
||||
} finally {
|
||||
size = rc.position();
|
||||
ctx.stats.readIdxBytes += size;
|
||||
ctx.stats.readIdxMicros += elapsedMicros(start);
|
||||
ctx.stats.readBitmapIdxBytes += size;
|
||||
ctx.stats.readBitmapIdxMicros += elapsedMicros(start);
|
||||
}
|
||||
bitmapIndex = bmidx;
|
||||
return new DfsBlockCache.Ref<>(
|
||||
|
|
|
@ -19,18 +19,39 @@ public static class Accumulator {
|
|||
/** Number of times the reader explicitly called scanPacks. */
|
||||
long scanPacks;
|
||||
|
||||
/** Total number of cache hits for pack indexes. */
|
||||
long idxCacheHit;
|
||||
|
||||
/** Total number of cache hits for reverse indexes. */
|
||||
long ridxCacheHit;
|
||||
|
||||
/** Total number of cache hits for bitmap indexes. */
|
||||
long bitmapCacheHit;
|
||||
|
||||
/** Total number of complete pack indexes read into memory. */
|
||||
long readIdx;
|
||||
|
||||
/** Total number of complete bitmap indexes read into memory. */
|
||||
long readBitmap;
|
||||
|
||||
/** Total number of bytes read from indexes. */
|
||||
/** Total number of reverse indexes added into memory. */
|
||||
long readReverseIdx;
|
||||
|
||||
/** Total number of bytes read from pack indexes. */
|
||||
long readIdxBytes;
|
||||
|
||||
/** Total microseconds spent reading pack or bitmap indexes. */
|
||||
/** Total microseconds spent reading pack indexes. */
|
||||
long readIdxMicros;
|
||||
|
||||
/** Total microseconds spent creating reverse indexes. */
|
||||
long readReverseIdxMicros;
|
||||
|
||||
/** Total number of bytes read from bitmap indexes. */
|
||||
long readBitmapIdxBytes;
|
||||
|
||||
/** Total microseconds spent reading bitmap indexes. */
|
||||
long readBitmapIdxMicros;
|
||||
|
||||
/** Total number of block cache hits. */
|
||||
long blockCacheHit;
|
||||
|
||||
|
@ -74,6 +95,33 @@ public long getScanPacks() {
|
|||
return stats.scanPacks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of pack index cache hits.
|
||||
*
|
||||
* @return total number of pack index cache hits.
|
||||
*/
|
||||
public long getPackIndexCacheHits() {
|
||||
return stats.idxCacheHit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of reverse index cache hits.
|
||||
*
|
||||
* @return total number of reverse index cache hits.
|
||||
*/
|
||||
public long getReverseIndexCacheHits() {
|
||||
return stats.ridxCacheHit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of bitmap index cache hits.
|
||||
*
|
||||
* @return total number of bitmap index cache hits.
|
||||
*/
|
||||
public long getBitmapIndexCacheHits() {
|
||||
return stats.bitmapCacheHit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of complete pack indexes read into memory.
|
||||
*
|
||||
|
@ -83,6 +131,15 @@ public long getReadPackIndexCount() {
|
|||
return stats.readIdx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of times the reverse index was computed.
|
||||
*
|
||||
* @return total number of reverse index was computed.
|
||||
*/
|
||||
public long getReadReverseIndexCount() {
|
||||
return stats.readReverseIdx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of complete bitmap indexes read into memory.
|
||||
*
|
||||
|
@ -93,23 +150,50 @@ public long getReadBitmapIndexCount() {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get total number of bytes read from indexes.
|
||||
* Get total number of bytes read from pack indexes.
|
||||
*
|
||||
* @return total number of bytes read from indexes.
|
||||
* @return total number of bytes read from pack indexes.
|
||||
*/
|
||||
public long getReadIndexBytes() {
|
||||
return stats.readIdxBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total microseconds spent reading pack or bitmap indexes.
|
||||
* Get total microseconds spent reading pack indexes.
|
||||
*
|
||||
* @return total microseconds spent reading pack or bitmap indexes.
|
||||
* @return total microseconds spent reading pack indexes.
|
||||
*/
|
||||
public long getReadIndexMicros() {
|
||||
return stats.readIdxMicros;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total microseconds spent creating reverse indexes.
|
||||
*
|
||||
* @return total microseconds spent creating reverse indexes.
|
||||
*/
|
||||
public long getReadReverseIndexMicros() {
|
||||
return stats.readReverseIdxMicros;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of bytes read from bitmap indexes.
|
||||
*
|
||||
* @return total number of bytes read from bitmap indexes.
|
||||
*/
|
||||
public long getReadBitmapIndexBytes() {
|
||||
return stats.readBitmapIdxBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total microseconds spent reading bitmap indexes.
|
||||
*
|
||||
* @return total microseconds spent reading bitmap indexes.
|
||||
*/
|
||||
public long getReadBitmapIndexMicros() {
|
||||
return stats.readBitmapIdxMicros;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total number of block cache hits.
|
||||
*
|
||||
|
|
Loading…
Reference in New Issue