motiejus
/
jgit
1
Fork 0
Code Releases Activity

Merge changes I1217f5f2,Iba037e0e,I61e6c93c,I6304d1cb 

* changes:
  dfs: Take size as long instead of int
  dfs: Read at the aligned position
  dfs: Add a position argument
  dfs: Move the deeply nested code to its own method
This commit is contained in:
David Pursehouse 2019-08-30 07:09:51 -04:00 committed by Gerrit Code Review @ Eclipse.org
parent 14ff64b587 c0cfdcd2f1
commit 2f751c34e1
2 changed files with 115 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java
View File

@ -412,8 +412,7 @@ DfsBlock getOrLoad(BlockBasedFile file, long position, DfsReader ctx,
getStat(statMiss, key).incrementAndGet(); getStat(statMiss, key).incrementAndGet();
boolean credit = true; boolean credit = true;
try { try {
v = file.readOneBlock(requestedPosition, ctx, v = file.readOneBlock(position, ctx, fileChannel.get());
fileChannel.get());
credit = false; credit = false;
} finally { } finally {
if (credit) { if (credit) {
@ -450,7 +449,7 @@ DfsBlock getOrLoad(BlockBasedFile file, long position, DfsReader ctx,
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private void reserveSpace(int reserve, DfsStreamKey key) { private void reserveSpace(long reserve, DfsStreamKey key) {
clockLock.lock(); clockLock.lock();
try { try {
long live = LongStream.of(getCurrentSize()).sum() + reserve; long live = LongStream.of(getCurrentSize()).sum() + reserve;
@ -487,7 +486,7 @@ private void reserveSpace(int reserve, DfsStreamKey key) {
} }
} }
private void creditSpace(int credit, DfsStreamKey key) { private void creditSpace(long credit, DfsStreamKey key) {
clockLock.lock(); clockLock.lock();
try { try {
getStat(liveBytes, key).addAndGet(-credit); getStat(liveBytes, key).addAndGet(-credit);
@ -497,7 +496,7 @@ private void creditSpace(int credit, DfsStreamKey key) {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private void addToClock(Ref ref, int credit) { private void addToClock(Ref ref, long credit) {
clockLock.lock(); clockLock.lock();
try { try {
if (credit != 0) { if (credit != 0) {
@ -521,17 +520,20 @@ void put(DfsBlock v) {
* *
* @param key * @param key
* the stream key of the pack. * the stream key of the pack.
* @param position
* the position in the key. The default should be 0.
* @param loader * @param loader
* the function to load the reference. * the function to load the reference.
* @return the object reference. * @return the object reference.
* @throws IOException * @throws IOException
* the reference was not in the cache and could not be loaded. * the reference was not in the cache and could not be loaded.
*/ */
<T> Ref<T> getOrLoadRef(DfsStreamKey key, RefLoader<T> loader) <T> Ref<T> getOrLoadRef(
DfsStreamKey key, long position, RefLoader<T> loader)
throws IOException { throws IOException {
int slot = slot(key, 0); int slot = slot(key, position);
HashEntry e1 = table.get(slot); HashEntry e1 = table.get(slot);
Ref<T> ref = scanRef(e1, key, 0); Ref<T> ref = scanRef(e1, key, position);
if (ref != null) { if (ref != null) {
getStat(statHit, key).incrementAndGet(); getStat(statHit, key).incrementAndGet();
return ref; return ref;
@ -543,7 +545,7 @@ <T> Ref<T> getOrLoadRef(DfsStreamKey key, RefLoader<T> loader)
try { try {
HashEntry e2 = table.get(slot); HashEntry e2 = table.get(slot);
if (e2 != e1) { if (e2 != e1) {
ref = scanRef(e2, key, 0); ref = scanRef(e2, key, position);
if (ref != null) { if (ref != null) {
getStat(statHit, key).incrementAndGet(); getStat(statHit, key).incrementAndGet();
return ref; return ref;
@ -574,10 +576,10 @@ <T> Ref<T> getOrLoadRef(DfsStreamKey key, RefLoader<T> loader)
} }
<T> Ref<T> putRef(DfsStreamKey key, long size, T v) { <T> Ref<T> putRef(DfsStreamKey key, long size, T v) {
return put(key, 0, (int) Math.min(size, Integer.MAX_VALUE), v); return put(key, 0, size, v);
} }
<T> Ref<T> put(DfsStreamKey key, long pos, int size, T v) { <T> Ref<T> put(DfsStreamKey key, long pos, long size, T v) {
int slot = slot(key, pos); int slot = slot(key, pos);
HashEntry e1 = table.get(slot); HashEntry e1 = table.get(slot);
Ref<T> ref = scanRef(e1, key, pos); Ref<T> ref = scanRef(e1, key, pos);
@ -720,12 +722,12 @@ private static final class HashEntry {
static final class Ref<T> { static final class Ref<T> {
final DfsStreamKey key; final DfsStreamKey key;
final long position; final long position;
final int size; final long size;
volatile T value; volatile T value;
Ref next; Ref next;
volatile boolean hot; volatile boolean hot;
Ref(DfsStreamKey key, long position, int size, T v) { Ref(DfsStreamKey key, long position, long size, T v) {
this.key = key; this.key = key;
this.position = position; this.position = position;
this.size = size; this.size = size;

185
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
View File

@ -89,6 +89,7 @@
*/ */
public final class DfsPackFile extends BlockBasedFile { public final class DfsPackFile extends BlockBasedFile {
private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8; private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8;
private static final long REF_POSITION = 0;
/** /**
* Lock for initialization of {@link #index} and {@link #corruptObjects}. * Lock for initialization of {@link #index} and {@link #corruptObjects}.
@ -194,45 +195,10 @@ private PackIndex idx(DfsReader ctx) throws IOException {
try { try {
DfsStreamKey idxKey = desc.getStreamKey(INDEX); DfsStreamKey idxKey = desc.getStreamKey(INDEX);
DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(idxKey, DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
() -> { idxKey,
try { REF_POSITION,
ctx.stats.readIdx++; () -> loadPackIndex(ctx, idxKey));
long start = System.nanoTime();
try (ReadableChannel rc = ctx.db.openFile(desc,
INDEX)) {
InputStream in = Channels
.newInputStream(rc);
int wantSize = 8192;
int bs = rc.blockSize();
if (0 < bs && bs < wantSize) {
bs = (wantSize / bs) * bs;
} else if (bs <= 0) {
bs = wantSize;
}
PackIndex idx = PackIndex.read(
new BufferedInputStream(in, bs));
int sz = (int) Math.min(
idx.getObjectCount() * REC_SIZE,
Integer.MAX_VALUE);
ctx.stats.readIdxBytes += rc.position();
index = idx;
return new DfsBlockCache.Ref<>(idxKey, 0,
sz, idx);
} finally {
ctx.stats.readIdxMicros += elapsedMicros(
start);
}
} catch (EOFException e) {
throw new IOException(MessageFormat.format(
DfsText.get().shortReadOfIndex,
desc.getFileName(INDEX)), e);
} catch (IOException e) {
throw new IOException(MessageFormat.format(
DfsText.get().cannotReadIndex,
desc.getFileName(INDEX)), e);
}
});
PackIndex idx = idxref.get(); PackIndex idx = idxref.get();
if (index == null && idx != null) { if (index == null && idx != null) {
index = idx; index = idx;
@ -267,44 +233,10 @@ PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
PackIndex idx = idx(ctx); PackIndex idx = idx(ctx);
PackReverseIndex revidx = getReverseIdx(ctx); PackReverseIndex revidx = getReverseIdx(ctx);
DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX); DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
DfsBlockCache.Ref<PackBitmapIndex> idxref = cache DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
.getOrLoadRef(bitmapKey, () -> { bitmapKey,
ctx.stats.readBitmap++; REF_POSITION,
long start = System.nanoTime(); () -> loadBitmapIndex(ctx, bitmapKey, idx, revidx));
try (ReadableChannel rc = ctx.db.openFile(desc,
BITMAP_INDEX)) {
long size;
PackBitmapIndex bmidx;
try {
InputStream in = Channels.newInputStream(rc);
int wantSize = 8192;
int bs = rc.blockSize();
if (0 < bs && bs < wantSize) {
bs = (wantSize / bs) * bs;
} else if (bs <= 0) {
bs = wantSize;
}
in = new BufferedInputStream(in, bs);
bmidx = PackBitmapIndex.read(in, idx, revidx);
} finally {
size = rc.position();
ctx.stats.readIdxBytes += size;
ctx.stats.readIdxMicros += elapsedMicros(start);
}
int sz = (int) Math.min(size, Integer.MAX_VALUE);
bitmapIndex = bmidx;
return new DfsBlockCache.Ref<>(bitmapKey, 0, sz,
bmidx);
} catch (EOFException e) {
throw new IOException(MessageFormat.format(
DfsText.get().shortReadOfIndex,
desc.getFileName(BITMAP_INDEX)), e);
} catch (IOException e) {
throw new IOException(MessageFormat.format(
DfsText.get().cannotReadIndex,
desc.getFileName(BITMAP_INDEX)), e);
}
});
PackBitmapIndex bmidx = idxref.get(); PackBitmapIndex bmidx = idxref.get();
if (bitmapIndex == null && bmidx != null) { if (bitmapIndex == null && bmidx != null) {
bitmapIndex = bmidx; bitmapIndex = bmidx;
@ -326,14 +258,10 @@ PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
PackIndex idx = idx(ctx); PackIndex idx = idx(ctx);
DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex( DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
desc.getStreamKey(INDEX)); desc.getStreamKey(INDEX));
DfsBlockCache.Ref<PackReverseIndex> revref = cache DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
.getOrLoadRef(revKey, () -> { revKey,
PackReverseIndex revidx = new PackReverseIndex(idx); REF_POSITION,
int sz = (int) Math.min(idx.getObjectCount() * 8, () -> loadReverseIdx(ctx, revKey, idx));
Integer.MAX_VALUE);
reverseIndex = revidx;
return new DfsBlockCache.Ref<>(revKey, 0, sz, revidx);
});
PackReverseIndex revidx = revref.get(); PackReverseIndex revidx = revref.get();
if (reverseIndex == null && revidx != null) { if (reverseIndex == null && revidx != null) {
reverseIndex = revidx; reverseIndex = revidx;
@ -1091,4 +1019,91 @@ private void setCorrupt(long offset) {
list.add(offset); list.add(offset);
} }
} }
private DfsBlockCache.Ref<PackIndex> loadPackIndex(
DfsReader ctx, DfsStreamKey idxKey) throws IOException {
try {
ctx.stats.readIdx++;
long start = System.nanoTime();
try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) {
InputStream in = Channels.newInputStream(rc);
int wantSize = 8192;
int bs = rc.blockSize();
if (0 < bs && bs < wantSize) {
bs = (wantSize / bs) * bs;
} else if (bs <= 0) {
bs = wantSize;
}
PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs));
ctx.stats.readIdxBytes += rc.position();
index = idx;
return new DfsBlockCache.Ref<>(
idxKey,
REF_POSITION,
idx.getObjectCount() * REC_SIZE,
idx);
} finally {
ctx.stats.readIdxMicros += elapsedMicros(start);
}
} catch (EOFException e) {
throw new IOException(MessageFormat.format(
DfsText.get().shortReadOfIndex,
desc.getFileName(INDEX)), e);
} catch (IOException e) {
throw new IOException(MessageFormat.format(
DfsText.get().cannotReadIndex,
desc.getFileName(INDEX)), e);
}
}
private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
DfsReader ctx, DfsStreamKey revKey, PackIndex idx) {
PackReverseIndex revidx = new PackReverseIndex(idx);
reverseIndex = revidx;
return new DfsBlockCache.Ref<>(
revKey,
REF_POSITION,
idx.getObjectCount() * 8,
revidx);
}
private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(
DfsReader ctx,
DfsStreamKey bitmapKey,
PackIndex idx,
PackReverseIndex revidx) throws IOException {
ctx.stats.readBitmap++;
long start = System.nanoTime();
try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) {
long size;
PackBitmapIndex bmidx;
try {
InputStream in = Channels.newInputStream(rc);
int wantSize = 8192;
int bs = rc.blockSize();
if (0 < bs && bs < wantSize) {
bs = (wantSize / bs) * bs;
} else if (bs <= 0) {
bs = wantSize;
}
in = new BufferedInputStream(in, bs);
bmidx = PackBitmapIndex.read(in, idx, revidx);
} finally {
size = rc.position();
ctx.stats.readIdxBytes += size;
ctx.stats.readIdxMicros += elapsedMicros(start);
}
bitmapIndex = bmidx;
return new DfsBlockCache.Ref<>(
bitmapKey, REF_POSITION, size, bmidx);
} catch (EOFException e) {
throw new IOException(MessageFormat.format(
DfsText.get().shortReadOfIndex,
desc.getFileName(BITMAP_INDEX)), e);
} catch (IOException e) {
throw new IOException(MessageFormat.format(
DfsText.get().cannotReadIndex,
desc.getFileName(BITMAP_INDEX)), e);
}
}
} }