From 53fb027284af07c0344d3da6187a08afbfed59f0 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 1 Apr 2011 20:42:06 -0400 Subject: [PATCH] Make DeltaBaseCache per-ObjectReader The 'Counting objects' phase of PackWriter requires good hit rates from the DeltaBaseCache while walking trees, the deltas need to find their bases in the cache in order to inflate in a reasonable time. If JGit is running in a multi-threaded server, such as Gerrit Code Review, each thread needs its own DeltaBaseCache to prevent one thread from evicting the other thread's relevant bases. Move the cache to be per-ObjectReader, lazily allocated when required by a PackFile. Change-Id: If9d5ed06728e813632ae96dcfb811f4860b276e8 Signed-off-by: Shawn O. Pearce --- .../jgit/storage/file/DeltaBaseCache.java | 69 +++++++++---------- .../eclipse/jgit/storage/file/PackFile.java | 7 +- .../jgit/storage/file/WindowCursor.java | 9 +++ 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java index 8b548242b..0b2393435 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/DeltaBaseCache.java @@ -54,35 +54,36 @@ private static int hash(final long position) { return (((int) position) << 22) >>> 22; } - private static int maxByteCount; + private static volatile int defaultMaxByteCount; - private static final Slot[] cache; + private final int maxByteCount; - private static Slot lruHead; + private final Slot[] cache; - private static Slot lruTail; + private Slot lruHead; - private static int openByteCount; + private Slot lruTail; + + private int openByteCount; static { DEAD = new SoftReference(null); - maxByteCount = new WindowCacheConfig().getDeltaBaseCacheLimit(); + reconfigure(new WindowCacheConfig()); + } + static void reconfigure(WindowCacheConfig cfg) { + defaultMaxByteCount = cfg.getDeltaBaseCacheLimit(); + } + + DeltaBaseCache() { + maxByteCount = defaultMaxByteCount; cache = new Slot[CACHE_SZ]; - for (int i = 0; i < CACHE_SZ; i++) - cache[i] = new Slot(); } - static synchronized void reconfigure(final WindowCacheConfig cfg) { - final int dbLimit = cfg.getDeltaBaseCacheLimit(); - if (maxByteCount != dbLimit) { - maxByteCount = dbLimit; - releaseMemory(); - } - } - - static synchronized Entry get(final PackFile pack, final long position) { - final Slot e = cache[hash(position)]; + Entry get(final PackFile pack, final long position) { + Slot e = cache[hash(position)]; + if (e == null) + return null; if (e.provider == pack && e.position == position) { final Entry buf = e.data.get(); if (buf != null) { @@ -93,13 +94,18 @@ static synchronized Entry get(final PackFile pack, final long position) { return null; } - static synchronized void store(final PackFile pack, final long position, + void store(final PackFile pack, final long position, final byte[] data, final int objectType) { if (data.length > maxByteCount) return; // Too large to cache. - final Slot e = cache[hash(position)]; - clearEntry(e); + Slot e = cache[hash(position)]; + if (e == null) { + e = new Slot(); + cache[hash(position)] = e; + } else { + clearEntry(e); + } openByteCount += data.length; releaseMemory(); @@ -111,7 +117,7 @@ static synchronized void store(final PackFile pack, final long position, moveToHead(e); } - private static void releaseMemory() { + private void releaseMemory() { while (openByteCount > maxByteCount && lruTail != null) { final Slot currOldest = lruTail; final Slot nextOldest = currOldest.lruPrev; @@ -128,16 +134,7 @@ private static void releaseMemory() { } } - static synchronized void purge(final PackFile file) { - for (final Slot e : cache) { - if (e.provider == file) { - clearEntry(e); - unlink(e); - } - } - } - - private static void moveToHead(final Slot e) { + private void moveToHead(final Slot e) { unlink(e); e.lruPrev = null; e.lruNext = lruHead; @@ -148,7 +145,7 @@ private static void moveToHead(final Slot e) { lruHead = e; } - private static void unlink(final Slot e) { + private void unlink(final Slot e) { final Slot prev = e.lruPrev; final Slot next = e.lruNext; if (prev != null) @@ -157,17 +154,13 @@ private static void unlink(final Slot e) { next.lruPrev = prev; } - private static void clearEntry(final Slot e) { + private void clearEntry(final Slot e) { openByteCount -= e.sz; e.provider = null; e.data = DEAD; e.sz = 0; } - private DeltaBaseCache() { - throw new UnsupportedOperationException(); - } - static class Entry { final byte[] data; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java index 959bf4334..80a096633 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java @@ -236,7 +236,6 @@ void resolve(Set matches, AbbreviatedObjectId id, int matchLimit) * Close the resources utilized by this repository */ public void close() { - DeltaBaseCache.purge(this); WindowCache.purge(this); synchronized (this) { loadedIdx = null; @@ -723,7 +722,7 @@ ObjectLoader load(final WindowCursor curs, long pos) if (sz != delta.deltaSize) break SEARCH; - DeltaBaseCache.Entry e = DeltaBaseCache.get(this, base); + DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base); if (e != null) { type = e.type; data = e.data; @@ -741,7 +740,7 @@ ObjectLoader load(final WindowCursor curs, long pos) if (sz != delta.deltaSize) break SEARCH; - DeltaBaseCache.Entry e = DeltaBaseCache.get(this, base); + DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base); if (e != null) { type = e.type; data = e.data; @@ -769,7 +768,7 @@ ObjectLoader load(final WindowCursor curs, long pos) if (cached) cached = false; else if (delta.next == null) - DeltaBaseCache.store(this, delta.basePos, data, type); + curs.getDeltaBaseCache().store(this, delta.basePos, data, type); pos = delta.deltaPos; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java index e2dcae82b..b3b93b70c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java @@ -83,12 +83,20 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { private ByteWindow window; + private DeltaBaseCache baseCache; + final FileObjectDatabase db; WindowCursor(FileObjectDatabase db) { this.db = db; } + DeltaBaseCache getDeltaBaseCache() { + if (baseCache == null) + baseCache = new DeltaBaseCache(); + return baseCache; + } + @Override public ObjectReader newReader() { return new WindowCursor(db); @@ -334,6 +342,7 @@ int getStreamFileThreshold() { /** Release the current window cursor. */ public void release() { window = null; + baseCache = null; try { InflaterCache.release(inf); } finally {