diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index ef0b80c11..6fff656e7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -322,6 +322,7 @@ DfsBlock getOrLoad(DfsPackFile pack, long position, DfsReader ctx) HashEntry e1 = table.get(slot); DfsBlock v = scan(e1, key, position); if (v != null) { + ctx.stats.blockCacheHit++; statHit.incrementAndGet(); return v; } @@ -334,6 +335,7 @@ DfsBlock getOrLoad(DfsPackFile pack, long position, DfsReader ctx) if (e2 != e1) { v = scan(e2, key, position); if (v != null) { + ctx.stats.blockCacheHit++; statHit.incrementAndGet(); creditSpace(blockSize); return v; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index f15d427f8..ae2e7e412 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -251,6 +251,8 @@ private PackIndex idx(DfsReader ctx) throws IOException { PackIndex idx; try { + ctx.stats.readIdx++; + long start = System.nanoTime(); ReadableChannel rc = ctx.db.openFile(packDesc, INDEX); try { InputStream in = Channels.newInputStream(rc); @@ -260,10 +262,11 @@ private PackIndex idx(DfsReader ctx) throws IOException { bs = (wantSize / bs) * bs; else if (bs <= 0) bs = wantSize; - in = new BufferedInputStream(in, bs); - idx = PackIndex.read(in); + idx = PackIndex.read(new BufferedInputStream(in, bs)); + ctx.stats.readIdxBytes += rc.position(); } finally { rc.close(); + ctx.stats.readIdxMicros += elapsedMicros(start); } } catch (EOFException e) { invalid = true; @@ -286,6 +289,10 @@ else if (bs <= 0) } } + private static long elapsedMicros(long start) { + return (System.nanoTime() - start) / 1000L; + } + final boolean isGarbage() { return packDesc.getPackSource() == UNREACHABLE_GARBAGE; } @@ -314,6 +321,8 @@ PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException { long size; PackBitmapIndex idx; try { + ctx.stats.readBitmap++; + long start = System.nanoTime(); ReadableChannel rc = ctx.db.openFile(packDesc, BITMAP_INDEX); try { InputStream in = Channels.newInputStream(rc); @@ -329,6 +338,8 @@ else if (bs <= 0) } finally { size = rc.position(); rc.close(); + ctx.stats.readIdxBytes += size; + ctx.stats.readIdxMicros += elapsedMicros(start); } } catch (EOFException e) { IOException e2 = new IOException(MessageFormat.format( @@ -777,6 +788,8 @@ DfsBlock readOneBlock(long pos, DfsReader ctx) if (invalid) throw new PackInvalidException(getPackName()); + ctx.stats.readBlock++; + long start = System.nanoTime(); ReadableChannel rc = ctx.db.openFile(packDesc, PACK); try { int size = blockSize(rc); @@ -803,6 +816,7 @@ DfsBlock readOneBlock(long pos, DfsReader ctx) byte[] buf = new byte[size]; rc.position(pos); int cnt = read(rc, ByteBuffer.wrap(buf, 0, size)); + ctx.stats.readBlockBytes += cnt; if (cnt != size) { if (0 <= len) { throw new EOFException(MessageFormat.format( @@ -824,10 +838,10 @@ DfsBlock readOneBlock(long pos, DfsReader ctx) length = len = rc.size(); } - DfsBlock v = new DfsBlock(key, pos, buf); - return v; + return new DfsBlock(key, pos, buf); } finally { rc.close(); + ctx.stats.readBlockMicros += elapsedMicros(start); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 755b16312..d4e7e86cc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -104,14 +104,12 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs { /** Database this reader loads objects from. */ final DfsObjDatabase db; + final DfsReaderIoStats.Accumulator stats = new DfsReaderIoStats.Accumulator(); + private Inflater inf; - private DfsBlock block; - private DeltaBaseCache baseCache; - private DfsPackFile last; - private boolean avoidUnreachable; DfsReader(DfsObjDatabase db) { @@ -170,6 +168,7 @@ public Collection resolve(AbbreviatedObjectId id) PackList packList = db.getPackList(); resolveImpl(packList, id, matches); if (matches.size() < MAX_RESOLVE_MATCHES && packList.dirty()) { + stats.scanPacks++; resolveImpl(db.scanPacks(packList), id, matches); } return matches; @@ -198,6 +197,7 @@ public boolean has(AnyObjectId objectId) throws IOException { if (hasImpl(packList, objectId)) { return true; } else if (packList.dirty()) { + stats.scanPacks++; return hasImpl(db.scanPacks(packList), objectId); } return false; @@ -234,6 +234,7 @@ public ObjectLoader open(AnyObjectId objectId, int typeHint) return checkType(ldr, objectId, typeHint); } if (packList.dirty()) { + stats.scanPacks++; ldr = openImpl(db.scanPacks(packList), objectId); if (ldr != null) { return checkType(ldr, objectId, typeHint); @@ -316,6 +317,7 @@ private Iterable> findAll( List> r = new ArrayList<>(); findAllImpl(packList, pending, r); if (!pending.isEmpty() && packList.dirty()) { + stats.scanPacks++; findAllImpl(db.scanPacks(packList), pending, r); } for (T t : pending) { @@ -452,7 +454,6 @@ public AsyncObjectSizeQueue getObjectSize( final IOException findAllError = error; return new AsyncObjectSizeQueue() { private FoundObject cur; - private long sz; @Override @@ -718,9 +719,10 @@ int inflate(DfsPackFile pack, long position, byte[] dstbuf, for (int dstoff = 0;;) { int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff); dstoff += n; - if (inf.finished() || (headerOnly && dstoff == dstbuf.length)) + if (inf.finished() || (headerOnly && dstoff == dstbuf.length)) { + stats.inflatedBytes += dstoff; return dstoff; - if (inf.needsInput()) { + } else if (inf.needsInput()) { pin(pack, position); position += block.setInput(position, inf); } else if (n == 0) @@ -764,6 +766,11 @@ void unpin() { block = null; } + /** @return IO statistics accumulated by this reader. */ + public DfsReaderIoStats getIoStats() { + return new DfsReaderIoStats(stats); + } + /** Release the current window cursor. */ @Override public void close() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java new file mode 100644 index 000000000..9a174c81d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.dfs; + +/** IO statistics for a {@link DfsReader}. */ +public class DfsReaderIoStats { + /** POJO to accumulate IO statistics. */ + public static class Accumulator { + /** Number of times the reader explicitly called scanPacks. */ + long scanPacks; + + /** Total number of complete pack indexes read into memory. */ + long readIdx; + + /** Total number of complete bitmap indexes read into memory. */ + long readBitmap; + + /** Total number of bytes read from indexes. */ + long readIdxBytes; + + /** Total microseconds spent reading pack or bitmap indexes. */ + long readIdxMicros; + + /** Total number of block cache hits. */ + long blockCacheHit; + + /** Total number of discrete blocks read from pack file(s). */ + long readBlock; + + /** Total number of compressed bytes read as block sized units. */ + long readBlockBytes; + + /** Total microseconds spent reading {@link #readBlock} blocks. */ + long readBlockMicros; + + /** Total number of bytes decompressed. */ + long inflatedBytes; + + Accumulator() { + } + } + + private final Accumulator stats; + + DfsReaderIoStats(Accumulator stats) { + this.stats = stats; + } + + /** @return number of times the reader explicitly called scanPacks. */ + public long getScanPacks() { + return stats.scanPacks; + } + + /** @return total number of complete pack indexes read into memory. */ + public long getReadPackIndexCount() { + return stats.readIdx; + } + + /** @return total number of complete bitmap indexes read into memory. */ + public long getReadBitmapIndexCount() { + return stats.readBitmap; + } + + /** @return total number of bytes read from indexes. */ + public long getReadIndexBytes() { + return stats.readIdxBytes; + } + + /** @return total microseconds spent reading pack or bitmap indexes. */ + public long getReadIndexMicros() { + return stats.readIdxMicros; + } + + /** @return total number of block cache hits. */ + public long getBlockCacheHits() { + return stats.blockCacheHit; + } + + /** @return total number of discrete blocks read from pack file(s). */ + public long getReadBlocksCount() { + return stats.readBlock; + } + + /** @return total number of compressed bytes read as block sized units. */ + public long getReadBlocksBytes() { + return stats.readBlockBytes; + } + + /** @return total microseconds spent reading blocks. */ + public long getReadBlocksMicros() { + return stats.readBlockMicros; + } + + /** @return total number of bytes decompressed. */ + public long getInflatedBytes() { + return stats.inflatedBytes; + } +}