Merge branch 'stable-5.3' into stable-5.4
* stable-5.3: Fix string format parameter for invalidRefAdvertisementLine WindowCache: add metric for cached bytes per repository pgm daemon: fallback to user and system config if no config specified WindowCache: add option to use strong refs to reference ByteWindows Replace usage of ArrayIndexOutOfBoundsException in treewalk Add config constants for WindowCache configuration options Change-Id: I12002dbfed9dff14fc6d2df9787d92eab5b1fa78
This commit is contained in:
commit
195b8fc94c
|
@ -44,6 +44,7 @@
|
|||
package org.eclipse.jgit.pgm;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.URISyntaxException;
|
||||
import java.text.MessageFormat;
|
||||
|
@ -51,12 +52,14 @@
|
|||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import org.eclipse.jgit.errors.ConfigInvalidException;
|
||||
import org.eclipse.jgit.internal.ketch.KetchLeader;
|
||||
import org.eclipse.jgit.internal.ketch.KetchLeaderCache;
|
||||
import org.eclipse.jgit.internal.ketch.KetchPreReceive;
|
||||
import org.eclipse.jgit.internal.ketch.KetchSystem;
|
||||
import org.eclipse.jgit.internal.ketch.KetchText;
|
||||
import org.eclipse.jgit.lib.Repository;
|
||||
import org.eclipse.jgit.lib.StoredConfig;
|
||||
import org.eclipse.jgit.pgm.internal.CLIText;
|
||||
import org.eclipse.jgit.storage.file.FileBasedConfig;
|
||||
import org.eclipse.jgit.storage.file.WindowCacheConfig;
|
||||
|
@ -68,6 +71,7 @@
|
|||
import org.eclipse.jgit.transport.resolver.ReceivePackFactory;
|
||||
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
|
||||
import org.eclipse.jgit.util.FS;
|
||||
import org.eclipse.jgit.util.SystemReader;
|
||||
import org.kohsuke.args4j.Argument;
|
||||
import org.kohsuke.args4j.Option;
|
||||
|
||||
|
@ -120,19 +124,20 @@ protected boolean requiresRepository() {
|
|||
@Override
|
||||
protected void run() throws Exception {
|
||||
PackConfig packConfig = new PackConfig();
|
||||
|
||||
if (configFile != null) {
|
||||
StoredConfig cfg;
|
||||
if (configFile == null) {
|
||||
cfg = getUserConfig();
|
||||
} else {
|
||||
if (!configFile.exists()) {
|
||||
throw die(MessageFormat.format(
|
||||
CLIText.get().configFileNotFound, //
|
||||
configFile.getAbsolutePath()));
|
||||
}
|
||||
|
||||
FileBasedConfig cfg = new FileBasedConfig(configFile, FS.DETECTED);
|
||||
cfg = new FileBasedConfig(configFile, FS.DETECTED);
|
||||
}
|
||||
cfg.load();
|
||||
new WindowCacheConfig().fromConfig(cfg).install();
|
||||
packConfig.fromConfig(cfg);
|
||||
}
|
||||
|
||||
int threads = packConfig.getThreads();
|
||||
if (threads <= 0)
|
||||
|
@ -172,6 +177,16 @@ protected void run() throws Exception {
|
|||
outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress()));
|
||||
}
|
||||
|
||||
private StoredConfig getUserConfig() throws IOException {
|
||||
StoredConfig userConfig = null;
|
||||
try {
|
||||
userConfig = SystemReader.getInstance().getUserConfig();
|
||||
} catch (ConfigInvalidException e) {
|
||||
throw die(e.getMessage());
|
||||
}
|
||||
return userConfig;
|
||||
}
|
||||
|
||||
private static DaemonService service(
|
||||
final org.eclipse.jgit.transport.Daemon d,
|
||||
final String n) {
|
||||
|
|
|
@ -53,6 +53,8 @@
|
|||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.eclipse.jgit.errors.CorruptObjectException;
|
||||
|
@ -66,9 +68,25 @@
|
|||
import org.eclipse.jgit.util.MutableInteger;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.junit.runners.Parameterized.Parameters;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class WindowCacheGetTest extends SampleDataRepositoryTestCase {
|
||||
private List<TestObject> toLoad;
|
||||
private WindowCacheConfig cfg;
|
||||
private boolean useStrongRefs;
|
||||
|
||||
@Parameters(name = "useStrongRefs={0}")
|
||||
public static Collection<Object[]> data() {
|
||||
return Arrays
|
||||
.asList(new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } });
|
||||
}
|
||||
|
||||
public WindowCacheGetTest(Boolean useStrongRef) {
|
||||
this.useStrongRefs = useStrongRef.booleanValue();
|
||||
}
|
||||
|
||||
@Override
|
||||
@Before
|
||||
|
@ -93,11 +111,12 @@ public void setUp() throws Exception {
|
|||
}
|
||||
}
|
||||
assertEquals(96, toLoad.size());
|
||||
cfg = new WindowCacheConfig();
|
||||
cfg.setPackedGitUseStrongRefs(useStrongRefs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCache_Defaults() throws IOException {
|
||||
WindowCacheConfig cfg = new WindowCacheConfig();
|
||||
cfg.install();
|
||||
doCacheTests();
|
||||
checkLimits(cfg);
|
||||
|
@ -122,7 +141,6 @@ public void testCache_Defaults() throws IOException {
|
|||
|
||||
@Test
|
||||
public void testCache_TooFewFiles() throws IOException {
|
||||
final WindowCacheConfig cfg = new WindowCacheConfig();
|
||||
cfg.setPackedGitOpenFiles(2);
|
||||
cfg.install();
|
||||
doCacheTests();
|
||||
|
@ -131,7 +149,6 @@ public void testCache_TooFewFiles() throws IOException {
|
|||
|
||||
@Test
|
||||
public void testCache_TooSmallLimit() throws IOException {
|
||||
final WindowCacheConfig cfg = new WindowCacheConfig();
|
||||
cfg.setPackedGitWindowSize(4096);
|
||||
cfg.setPackedGitLimit(4096);
|
||||
cfg.install();
|
||||
|
@ -142,26 +159,31 @@ public void testCache_TooSmallLimit() throws IOException {
|
|||
private static void checkLimits(WindowCacheConfig cfg) {
|
||||
final WindowCache cache = WindowCache.getInstance();
|
||||
WindowCacheStats s = cache.getStats();
|
||||
assertTrue(0 < s.getAverageLoadTime());
|
||||
assertTrue(0 < s.getOpenByteCount());
|
||||
assertTrue(0 < s.getOpenByteCount());
|
||||
assertTrue(0.0 < s.getAverageLoadTime());
|
||||
assertTrue(0 <= s.getEvictionCount());
|
||||
assertTrue(0 < s.getHitCount());
|
||||
assertTrue(0 < s.getHitRatio());
|
||||
assertTrue(1 > s.getHitRatio());
|
||||
assertTrue(0 < s.getLoadCount());
|
||||
assertTrue(0 <= s.getLoadFailureCount());
|
||||
assertTrue(0.0 <= s.getLoadFailureRatio());
|
||||
assertTrue(1 > s.getLoadFailureRatio());
|
||||
assertTrue(0 < s.getLoadSuccessCount());
|
||||
assertTrue(s.getOpenByteCount() <= cfg.getPackedGitLimit());
|
||||
assertTrue(s.getOpenFileCount() <= cfg.getPackedGitOpenFiles());
|
||||
assertTrue(0 <= s.getMissCount());
|
||||
assertTrue(0 <= s.getMissRatio());
|
||||
assertTrue(1 > s.getMissRatio());
|
||||
assertTrue(0 < s.getRequestCount());
|
||||
assertTrue(0 < s.getTotalLoadTime());
|
||||
assertTrue("average load time should be > 0",
|
||||
0 < s.getAverageLoadTime());
|
||||
assertTrue("open byte count should be > 0", 0 < s.getOpenByteCount());
|
||||
assertTrue("eviction count should be >= 0", 0 <= s.getEvictionCount());
|
||||
assertTrue("hit count should be > 0", 0 < s.getHitCount());
|
||||
assertTrue("hit ratio should be > 0", 0 < s.getHitRatio());
|
||||
assertTrue("hit ratio should be < 1", 1 > s.getHitRatio());
|
||||
assertTrue("load count should be > 0", 0 < s.getLoadCount());
|
||||
assertTrue("load failure count should be >= 0",
|
||||
0 <= s.getLoadFailureCount());
|
||||
assertTrue("load failure ratio should be >= 0",
|
||||
0.0 <= s.getLoadFailureRatio());
|
||||
assertTrue("load failure ratio should be < 1",
|
||||
1 > s.getLoadFailureRatio());
|
||||
assertTrue("load success count should be > 0",
|
||||
0 < s.getLoadSuccessCount());
|
||||
assertTrue("open byte count should be <= core.packedGitLimit",
|
||||
s.getOpenByteCount() <= cfg.getPackedGitLimit());
|
||||
assertTrue("open file count should be <= core.packedGitOpenFiles",
|
||||
s.getOpenFileCount() <= cfg.getPackedGitOpenFiles());
|
||||
assertTrue("miss success count should be >= 0", 0 <= s.getMissCount());
|
||||
assertTrue("miss ratio should be > 0", 0 <= s.getMissRatio());
|
||||
assertTrue("miss ratio should be < 1", 1 > s.getMissRatio());
|
||||
assertTrue("request count should be > 0", 0 < s.getRequestCount());
|
||||
assertTrue("total load time should be > 0", 0 < s.getTotalLoadTime());
|
||||
}
|
||||
|
||||
private void doCacheTests() throws IOException {
|
||||
|
|
|
@ -41,6 +41,36 @@
|
|||
<message_argument value="CONFIG_JMX_SECTION"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="CONFIG_KEY_PACKED_GIT_LIMIT"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="CONFIG_KEY_PACKED_GIT_MMAP"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="CONFIG_KEY_PACKED_GIT_OPENFILES"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="CONFIG_KEY_PACKED_GIT_USE_STRONGREFS"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="CONFIG_KEY_PACKED_GIT_WINDOWSIZE"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.9"/>
|
||||
|
@ -84,6 +114,20 @@
|
|||
</message_arguments>
|
||||
</filter>
|
||||
</resource>
|
||||
<resource path="src/org/eclipse/jgit/storage/file/WindowCacheConfig.java" type="org.eclipse.jgit.storage.file.WindowCacheConfig">
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="isPackedGitUseStrongRefs()"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
<filter id="1142947843">
|
||||
<message_arguments>
|
||||
<message_argument value="5.1.13"/>
|
||||
<message_argument value="setPackedGitUseStrongRefs(boolean)"/>
|
||||
</message_arguments>
|
||||
</filter>
|
||||
</resource>
|
||||
<resource path="src/org/eclipse/jgit/storage/file/WindowCacheStats.java" type="org.eclipse.jgit.storage.file.WindowCacheStats">
|
||||
<filter id="337809484">
|
||||
<message_arguments>
|
||||
|
|
|
@ -364,7 +364,7 @@ invalidPacketLineHeader=Invalid packet line header: {0}
|
|||
invalidPath=Invalid path: {0}
|
||||
invalidPurgeFactor=Invalid purgeFactor {0}, values have to be in range between 0 and 1
|
||||
invalidRedirectLocation=Invalid redirect location {0} -> {1}
|
||||
invalidRefAdvertisementLine=Invalid ref advertisement line: ''{1}''
|
||||
invalidRefAdvertisementLine=Invalid ref advertisement line: ''{0}''
|
||||
invalidReflogRevision=Invalid reflog revision: {0}
|
||||
invalidRefName=Invalid ref name: {0}
|
||||
invalidReftableBlock=Invalid reftable block
|
||||
|
|
|
@ -47,11 +47,16 @@
|
|||
import java.io.IOException;
|
||||
import java.lang.ref.ReferenceQueue;
|
||||
import java.lang.ref.SoftReference;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReferenceArray;
|
||||
import java.util.concurrent.atomic.LongAdder;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.eclipse.jgit.annotations.NonNull;
|
||||
import org.eclipse.jgit.internal.JGitText;
|
||||
|
@ -85,9 +90,16 @@
|
|||
* comprised of roughly 10% of the cache, and evicting the oldest accessed entry
|
||||
* within that window.
|
||||
* <p>
|
||||
* Entities created by the cache are held under SoftReferences, permitting the
|
||||
* Entities created by the cache are held under SoftReferences if option
|
||||
* {@code core.packedGitUseStrongRefs} is set to {@code false} in the git config
|
||||
* (this is the default) or by calling
|
||||
* {@link WindowCacheConfig#setPackedGitUseStrongRefs(boolean)}, permitting the
|
||||
* Java runtime's garbage collector to evict entries when heap memory gets low.
|
||||
* Most JREs implement a loose least recently used algorithm for this eviction.
|
||||
* When this option is set to {@code true} strong references are used which
|
||||
* means that Java gc cannot evict the WindowCache to reclaim memory. On the
|
||||
* other hand this provides more predictable performance since the cache isn't
|
||||
* flushed when used heap comes close to the maximum heap size.
|
||||
* <p>
|
||||
* The internal hash table does not expand at runtime, instead it is fixed in
|
||||
* size at cache creation time. The internal lock table used to gate load
|
||||
|
@ -104,19 +116,19 @@
|
|||
* for a given <code>(PackFile,position)</code> tuple.</li>
|
||||
* <li>For every <code>load()</code> invocation there is exactly one
|
||||
* {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a
|
||||
* SoftReference around the cached entity.</li>
|
||||
* SoftReference or a StrongReference around the cached entity.</li>
|
||||
* <li>For every Reference created by <code>createRef()</code> there will be
|
||||
* exactly one call to {@link #clear(Ref)} to cleanup any resources associated
|
||||
* exactly one call to {@link #clear(PageRef)} to cleanup any resources associated
|
||||
* with the (now expired) cached entity.</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* Therefore, it is safe to perform resource accounting increments during the
|
||||
* {@link #load(PackFile, long)} or
|
||||
* {@link #createRef(PackFile, long, ByteWindow)} methods, and matching
|
||||
* decrements during {@link #clear(Ref)}. Implementors may need to override
|
||||
* decrements during {@link #clear(PageRef)}. Implementors may need to override
|
||||
* {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional
|
||||
* accounting information into an implementation specific
|
||||
* {@link org.eclipse.jgit.internal.storage.file.WindowCache.Ref} subclass, as
|
||||
* {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as
|
||||
* the cached entity may have already been evicted by the JRE's garbage
|
||||
* collector.
|
||||
* <p>
|
||||
|
@ -176,18 +188,21 @@ static interface StatsRecorder {
|
|||
/**
|
||||
* Record files opened by cache
|
||||
*
|
||||
* @param count
|
||||
* @param delta
|
||||
* delta of number of files opened by cache
|
||||
*/
|
||||
void recordOpenFiles(int count);
|
||||
void recordOpenFiles(int delta);
|
||||
|
||||
/**
|
||||
* Record cached bytes
|
||||
*
|
||||
* @param count
|
||||
* @param pack
|
||||
* pack file the bytes are read from
|
||||
*
|
||||
* @param delta
|
||||
* delta of cached bytes
|
||||
*/
|
||||
void recordOpenBytes(int count);
|
||||
void recordOpenBytes(PackFile pack, int delta);
|
||||
|
||||
/**
|
||||
* Returns a snapshot of this recorder's stats. Note that this may be an
|
||||
|
@ -209,6 +224,7 @@ static class StatsRecorderImpl
|
|||
private final LongAdder evictionCount;
|
||||
private final LongAdder openFileCount;
|
||||
private final LongAdder openByteCount;
|
||||
private final Map<String, LongAdder> openByteCountPerRepository;
|
||||
|
||||
/**
|
||||
* Constructs an instance with all counts initialized to zero.
|
||||
|
@ -222,6 +238,7 @@ public StatsRecorderImpl() {
|
|||
evictionCount = new LongAdder();
|
||||
openFileCount = new LongAdder();
|
||||
openByteCount = new LongAdder();
|
||||
openByteCountPerRepository = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -252,13 +269,28 @@ public void recordEvictions(int count) {
|
|||
}
|
||||
|
||||
@Override
|
||||
public void recordOpenFiles(int count) {
|
||||
openFileCount.add(count);
|
||||
public void recordOpenFiles(int delta) {
|
||||
openFileCount.add(delta);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordOpenBytes(int count) {
|
||||
openByteCount.add(count);
|
||||
public void recordOpenBytes(PackFile pack, int delta) {
|
||||
openByteCount.add(delta);
|
||||
String repositoryId = repositoryId(pack);
|
||||
LongAdder la = openByteCountPerRepository
|
||||
.computeIfAbsent(repositoryId, k -> new LongAdder());
|
||||
la.add(delta);
|
||||
if (delta < 0) {
|
||||
openByteCountPerRepository.computeIfPresent(repositoryId,
|
||||
(k, v) -> v.longValue() == 0 ? null : v);
|
||||
}
|
||||
}
|
||||
|
||||
private static String repositoryId(PackFile pack) {
|
||||
// use repository's gitdir since packfile doesn't know its
|
||||
// repository
|
||||
return pack.getPackFile().getParentFile().getParentFile()
|
||||
.getParent();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -315,6 +347,15 @@ public void resetCounters() {
|
|||
totalLoadTime.reset();
|
||||
evictionCount.reset();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Long> getOpenByteCountPerRepository() {
|
||||
return Collections.unmodifiableMap(
|
||||
openByteCountPerRepository.entrySet().stream()
|
||||
.collect(Collectors.toMap(Map.Entry::getKey,
|
||||
e -> Long.valueOf(e.getValue().sum()),
|
||||
(u, v) -> v)));
|
||||
}
|
||||
}
|
||||
|
||||
private static final int bits(int newSize) {
|
||||
|
@ -390,8 +431,8 @@ static final void purge(PackFile pack) {
|
|||
cache.removeAll(pack);
|
||||
}
|
||||
|
||||
/** ReferenceQueue to cleanup released and garbage collected windows. */
|
||||
private final ReferenceQueue<ByteWindow> queue;
|
||||
/** cleanup released and/or garbage collected windows. */
|
||||
private final CleanupQueue queue;
|
||||
|
||||
/** Number of entries in {@link #table}. */
|
||||
private final int tableSize;
|
||||
|
@ -425,6 +466,8 @@ static final void purge(PackFile pack) {
|
|||
|
||||
private final StatsRecorderImpl mbean;
|
||||
|
||||
private boolean useStrongRefs;
|
||||
|
||||
private WindowCache(WindowCacheConfig cfg) {
|
||||
tableSize = tableSize(cfg);
|
||||
final int lockCount = lockCount(cfg);
|
||||
|
@ -433,7 +476,6 @@ private WindowCache(WindowCacheConfig cfg) {
|
|||
if (lockCount < 1)
|
||||
throw new IllegalArgumentException(JGitText.get().lockCountMustBeGreaterOrEqual1);
|
||||
|
||||
queue = new ReferenceQueue<>();
|
||||
clock = new AtomicLong(1);
|
||||
table = new AtomicReferenceArray<>(tableSize);
|
||||
locks = new Lock[lockCount];
|
||||
|
@ -455,6 +497,9 @@ else if (eb < 4)
|
|||
mmap = cfg.isPackedGitMMAP();
|
||||
windowSizeShift = bits(cfg.getPackedGitWindowSize());
|
||||
windowSize = 1 << windowSizeShift;
|
||||
useStrongRefs = cfg.isPackedGitUseStrongRefs();
|
||||
queue = useStrongRefs ? new StrongCleanupQueue(this)
|
||||
: new SoftCleanupQueue(this);
|
||||
|
||||
mbean = new StatsRecorderImpl();
|
||||
statsRecorder = mbean;
|
||||
|
@ -503,16 +548,18 @@ private ByteWindow load(PackFile pack, long offset) throws IOException {
|
|||
}
|
||||
}
|
||||
|
||||
private Ref createRef(PackFile p, long o, ByteWindow v) {
|
||||
final Ref ref = new Ref(p, o, v, queue);
|
||||
statsRecorder.recordOpenBytes(ref.size);
|
||||
private PageRef<ByteWindow> createRef(PackFile p, long o, ByteWindow v) {
|
||||
final PageRef<ByteWindow> ref = useStrongRefs
|
||||
? new StrongRef(p, o, v, queue)
|
||||
: new SoftRef(p, o, v, (SoftCleanupQueue) queue);
|
||||
statsRecorder.recordOpenBytes(ref.getPack(), ref.getSize());
|
||||
return ref;
|
||||
}
|
||||
|
||||
private void clear(Ref ref) {
|
||||
statsRecorder.recordOpenBytes(-ref.size);
|
||||
private void clear(PageRef<ByteWindow> ref) {
|
||||
statsRecorder.recordOpenBytes(ref.getPack(), -ref.getSize());
|
||||
statsRecorder.recordEvictions(1);
|
||||
close(ref.pack);
|
||||
close(ref.getPack());
|
||||
}
|
||||
|
||||
private void close(PackFile pack) {
|
||||
|
@ -577,7 +624,7 @@ private ByteWindow getOrLoad(PackFile pack, long position)
|
|||
}
|
||||
|
||||
v = load(pack, position);
|
||||
final Ref ref = createRef(pack, position, v);
|
||||
final PageRef<ByteWindow> ref = createRef(pack, position, v);
|
||||
hit(ref);
|
||||
for (;;) {
|
||||
final Entry n = new Entry(clean(e2), ref);
|
||||
|
@ -601,8 +648,8 @@ private ByteWindow getOrLoad(PackFile pack, long position)
|
|||
|
||||
private ByteWindow scan(Entry n, PackFile pack, long position) {
|
||||
for (; n != null; n = n.next) {
|
||||
final Ref r = n.ref;
|
||||
if (r.pack == pack && r.position == position) {
|
||||
final PageRef<ByteWindow> r = n.ref;
|
||||
if (r.getPack() == pack && r.getPosition() == position) {
|
||||
final ByteWindow v = r.get();
|
||||
if (v != null) {
|
||||
hit(r);
|
||||
|
@ -615,7 +662,7 @@ private ByteWindow scan(Entry n, PackFile pack, long position) {
|
|||
return null;
|
||||
}
|
||||
|
||||
private void hit(Ref r) {
|
||||
private void hit(PageRef r) {
|
||||
// We don't need to be 100% accurate here. Its sufficient that at least
|
||||
// one thread performs the increment. Any other concurrent access at
|
||||
// exactly the same time can simply use the same clock value.
|
||||
|
@ -625,7 +672,7 @@ private void hit(Ref r) {
|
|||
//
|
||||
final long c = clock.get();
|
||||
clock.compareAndSet(c, c + 1);
|
||||
r.lastAccess = c;
|
||||
r.setLastAccess(c);
|
||||
}
|
||||
|
||||
private void evict() {
|
||||
|
@ -639,7 +686,8 @@ private void evict() {
|
|||
for (Entry e = table.get(ptr); e != null; e = e.next) {
|
||||
if (e.dead)
|
||||
continue;
|
||||
if (old == null || e.ref.lastAccess < old.ref.lastAccess) {
|
||||
if (old == null || e.ref.getLastAccess() < old.ref
|
||||
.getLastAccess()) {
|
||||
old = e;
|
||||
slot = ptr;
|
||||
}
|
||||
|
@ -659,7 +707,7 @@ private void evict() {
|
|||
* <p>
|
||||
* This is a last-ditch effort to clear out the cache, such as before it
|
||||
* gets replaced by another cache that is configured differently. This
|
||||
* method tries to force every cached entry through {@link #clear(Ref)} to
|
||||
* method tries to force every cached entry through {@link #clear(PageRef)} to
|
||||
* ensure that resources are correctly accounted for and cleaned up by the
|
||||
* subclass. A concurrent reader loading entries while this method is
|
||||
* running may cause resource accounting failures.
|
||||
|
@ -692,7 +740,7 @@ private void removeAll(PackFile pack) {
|
|||
final Entry e1 = table.get(s);
|
||||
boolean hasDead = false;
|
||||
for (Entry e = e1; e != null; e = e.next) {
|
||||
if (e.ref.pack == pack) {
|
||||
if (e.ref.getPack() == pack) {
|
||||
e.kill();
|
||||
hasDead = true;
|
||||
} else if (e.dead)
|
||||
|
@ -705,20 +753,7 @@ private void removeAll(PackFile pack) {
|
|||
}
|
||||
|
||||
private void gc() {
|
||||
Ref r;
|
||||
while ((r = (Ref) queue.poll()) != null) {
|
||||
clear(r);
|
||||
|
||||
final int s = slot(r.pack, r.position);
|
||||
final Entry e1 = table.get(s);
|
||||
for (Entry n = e1; n != null; n = n.next) {
|
||||
if (n.ref == r) {
|
||||
n.dead = true;
|
||||
table.compareAndSet(s, e1, clean(e1));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
queue.gc();
|
||||
}
|
||||
|
||||
private int slot(PackFile pack, long position) {
|
||||
|
@ -731,7 +766,7 @@ private Lock lock(PackFile pack, long position) {
|
|||
|
||||
private static Entry clean(Entry top) {
|
||||
while (top != null && top.dead) {
|
||||
top.ref.enqueue();
|
||||
top.ref.kill();
|
||||
top = top.next;
|
||||
}
|
||||
if (top == null)
|
||||
|
@ -745,7 +780,7 @@ private static class Entry {
|
|||
final Entry next;
|
||||
|
||||
/** The referenced object. */
|
||||
final Ref ref;
|
||||
final PageRef<ByteWindow> ref;
|
||||
|
||||
/**
|
||||
* Marked true when ref.get() returns null and the ref is dead.
|
||||
|
@ -756,34 +791,275 @@ private static class Entry {
|
|||
*/
|
||||
volatile boolean dead;
|
||||
|
||||
Entry(Entry n, Ref r) {
|
||||
Entry(Entry n, PageRef<ByteWindow> r) {
|
||||
next = n;
|
||||
ref = r;
|
||||
}
|
||||
|
||||
final void kill() {
|
||||
dead = true;
|
||||
ref.enqueue();
|
||||
ref.kill();
|
||||
}
|
||||
}
|
||||
|
||||
private static interface PageRef<T> {
|
||||
/**
|
||||
* Returns this reference object's referent. If this reference object
|
||||
* has been cleared, either by the program or by the garbage collector,
|
||||
* then this method returns <code>null</code>.
|
||||
*
|
||||
* @return The object to which this reference refers, or
|
||||
* <code>null</code> if this reference object has been cleared
|
||||
*/
|
||||
T get();
|
||||
|
||||
/**
|
||||
* Kill this ref
|
||||
*
|
||||
* @return <code>true</code> if this reference object was successfully
|
||||
* killed; <code>false</code> if it was already killed
|
||||
*/
|
||||
boolean kill();
|
||||
|
||||
/**
|
||||
* Get the packfile the referenced cache page is allocated for
|
||||
*
|
||||
* @return the packfile the referenced cache page is allocated for
|
||||
*/
|
||||
PackFile getPack();
|
||||
|
||||
/**
|
||||
* Get the position of the referenced cache page in the packfile
|
||||
*
|
||||
* @return the position of the referenced cache page in the packfile
|
||||
*/
|
||||
long getPosition();
|
||||
|
||||
/**
|
||||
* Get size of cache page
|
||||
*
|
||||
* @return size of cache page
|
||||
*/
|
||||
int getSize();
|
||||
|
||||
/**
|
||||
* Get pseudo time of last access to this cache page
|
||||
*
|
||||
* @return pseudo time of last access to this cache page
|
||||
*/
|
||||
long getLastAccess();
|
||||
|
||||
/**
|
||||
* Set pseudo time of last access to this cache page
|
||||
*
|
||||
* @param time
|
||||
* pseudo time of last access to this cache page
|
||||
*/
|
||||
void setLastAccess(long time);
|
||||
|
||||
/**
|
||||
* Whether this is a strong reference.
|
||||
* @return {@code true} if this is a strong reference
|
||||
*/
|
||||
boolean isStrongRef();
|
||||
}
|
||||
|
||||
/** A soft reference wrapped around a cached object. */
|
||||
private static class Ref extends SoftReference<ByteWindow> {
|
||||
final PackFile pack;
|
||||
private static class SoftRef extends SoftReference<ByteWindow>
|
||||
implements PageRef<ByteWindow> {
|
||||
private final PackFile pack;
|
||||
|
||||
final long position;
|
||||
private final long position;
|
||||
|
||||
final int size;
|
||||
private final int size;
|
||||
|
||||
long lastAccess;
|
||||
private long lastAccess;
|
||||
|
||||
protected Ref(final PackFile pack, final long position,
|
||||
final ByteWindow v, final ReferenceQueue<ByteWindow> queue) {
|
||||
protected SoftRef(final PackFile pack, final long position,
|
||||
final ByteWindow v, final SoftCleanupQueue queue) {
|
||||
super(v, queue);
|
||||
this.pack = pack;
|
||||
this.position = position;
|
||||
this.size = v.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PackFile getPack() {
|
||||
return pack;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPosition() {
|
||||
return position;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLastAccess() {
|
||||
return lastAccess;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLastAccess(long time) {
|
||||
this.lastAccess = time;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean kill() {
|
||||
return enqueue();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isStrongRef() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** A strong reference wrapped around a cached object. */
|
||||
private static class StrongRef implements PageRef<ByteWindow> {
|
||||
private ByteWindow referent;
|
||||
|
||||
private final PackFile pack;
|
||||
|
||||
private final long position;
|
||||
|
||||
private final int size;
|
||||
|
||||
private long lastAccess;
|
||||
|
||||
private CleanupQueue queue;
|
||||
|
||||
protected StrongRef(final PackFile pack, final long position,
|
||||
final ByteWindow v, final CleanupQueue queue) {
|
||||
this.pack = pack;
|
||||
this.position = position;
|
||||
this.referent = v;
|
||||
this.size = v.size();
|
||||
this.queue = queue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PackFile getPack() {
|
||||
return pack;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPosition() {
|
||||
return position;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getSize() {
|
||||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLastAccess() {
|
||||
return lastAccess;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setLastAccess(long time) {
|
||||
this.lastAccess = time;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ByteWindow get() {
|
||||
return referent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean kill() {
|
||||
if (referent == null) {
|
||||
return false;
|
||||
}
|
||||
referent = null;
|
||||
return queue.enqueue(this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isStrongRef() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static interface CleanupQueue {
|
||||
boolean enqueue(PageRef<ByteWindow> r);
|
||||
void gc();
|
||||
}
|
||||
|
||||
private static class SoftCleanupQueue extends ReferenceQueue<ByteWindow>
|
||||
implements CleanupQueue {
|
||||
private final WindowCache wc;
|
||||
|
||||
SoftCleanupQueue(WindowCache cache) {
|
||||
this.wc = cache;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean enqueue(PageRef<ByteWindow> r) {
|
||||
// no need to explicitly add soft references which are enqueued by
|
||||
// the JVM
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void gc() {
|
||||
SoftRef r;
|
||||
while ((r = (SoftRef) poll()) != null) {
|
||||
wc.clear(r);
|
||||
|
||||
final int s = wc.slot(r.getPack(), r.getPosition());
|
||||
final Entry e1 = wc.table.get(s);
|
||||
for (Entry n = e1; n != null; n = n.next) {
|
||||
if (n.ref == r) {
|
||||
n.dead = true;
|
||||
wc.table.compareAndSet(s, e1, clean(e1));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class StrongCleanupQueue implements CleanupQueue {
|
||||
private final WindowCache wc;
|
||||
|
||||
private final ConcurrentLinkedQueue<PageRef<ByteWindow>> queue = new ConcurrentLinkedQueue<>();
|
||||
|
||||
StrongCleanupQueue(WindowCache wc) {
|
||||
this.wc = wc;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean enqueue(PageRef<ByteWindow> r) {
|
||||
if (queue.contains(r)) {
|
||||
return false;
|
||||
}
|
||||
return queue.add(r);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void gc() {
|
||||
PageRef<ByteWindow> r;
|
||||
while ((r = queue.poll()) != null) {
|
||||
wc.clear(r);
|
||||
|
||||
final int s = wc.slot(r.getPack(), r.getPosition());
|
||||
final Entry e1 = wc.table.get(s);
|
||||
for (Entry n = e1; n != null; n = n.next) {
|
||||
if (n.ref == r) {
|
||||
n.dead = true;
|
||||
wc.table.compareAndSet(s, e1, clean(e1));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class Lock {
|
||||
|
|
|
@ -230,6 +230,36 @@ public final class ConfigConstants {
|
|||
/** The "streamFileThreshold" key */
|
||||
public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = "streamFileThreshold";
|
||||
|
||||
/**
|
||||
* The "packedGitMmap" key
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public static final String CONFIG_KEY_PACKED_GIT_MMAP = "packedgitmmap";
|
||||
|
||||
/**
|
||||
* The "packedGitWindowSize" key
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public static final String CONFIG_KEY_PACKED_GIT_WINDOWSIZE = "packedgitwindowsize";
|
||||
|
||||
/**
|
||||
* The "packedGitLimit" key
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public static final String CONFIG_KEY_PACKED_GIT_LIMIT = "packedgitlimit";
|
||||
|
||||
/**
|
||||
* The "packedGitOpenFiles" key
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public static final String CONFIG_KEY_PACKED_GIT_OPENFILES = "packedgitopenfiles";
|
||||
|
||||
/**
|
||||
* The "packedGitUseStrongRefs" key
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public static final String CONFIG_KEY_PACKED_GIT_USE_STRONGREFS = "packedgitusestrongrefs";
|
||||
|
||||
/** The "remote" key */
|
||||
public static final String CONFIG_KEY_REMOTE = "remote";
|
||||
|
||||
|
|
|
@ -43,6 +43,15 @@
|
|||
|
||||
package org.eclipse.jgit.storage.file;
|
||||
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_LIMIT;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_MMAP;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_OPENFILES;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_WINDOWSIZE;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_TRESHOLD;
|
||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_USE_STRONGREFS;
|
||||
|
||||
import org.eclipse.jgit.internal.storage.file.WindowCache;
|
||||
import org.eclipse.jgit.lib.Config;
|
||||
import org.eclipse.jgit.storage.pack.PackConfig;
|
||||
|
@ -61,6 +70,8 @@ public class WindowCacheConfig {
|
|||
|
||||
private long packedGitLimit;
|
||||
|
||||
private boolean useStrongRefs;
|
||||
|
||||
private int packedGitWindowSize;
|
||||
|
||||
private boolean packedGitMMAP;
|
||||
|
@ -75,6 +86,7 @@ public class WindowCacheConfig {
|
|||
public WindowCacheConfig() {
|
||||
packedGitOpenFiles = 128;
|
||||
packedGitLimit = 10 * MB;
|
||||
useStrongRefs = false;
|
||||
packedGitWindowSize = 8 * KB;
|
||||
packedGitMMAP = false;
|
||||
deltaBaseCacheLimit = 10 * MB;
|
||||
|
@ -125,6 +137,31 @@ public void setPackedGitLimit(long newLimit) {
|
|||
packedGitLimit = newLimit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get whether the window cache should use strong references or
|
||||
* SoftReferences
|
||||
*
|
||||
* @return {@code true} if the window cache should use strong references,
|
||||
* otherwise it will use {@link java.lang.ref.SoftReference}s
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public boolean isPackedGitUseStrongRefs() {
|
||||
return useStrongRefs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set if the cache should use strong refs or soft refs
|
||||
*
|
||||
* @param useStrongRefs
|
||||
* if @{code true} the cache strongly references cache pages
|
||||
* otherwise it uses {@link java.lang.ref.SoftReference}s which
|
||||
* can be evicted by the Java gc if heap is almost full
|
||||
* @since 5.1.13
|
||||
*/
|
||||
public void setPackedGitUseStrongRefs(boolean useStrongRefs) {
|
||||
this.useStrongRefs = useStrongRefs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get size in bytes of a single window mapped or read in from the pack
|
||||
* file.
|
||||
|
@ -227,20 +264,23 @@ public void setStreamFileThreshold(int newLimit) {
|
|||
* @since 3.0
|
||||
*/
|
||||
public WindowCacheConfig fromConfig(Config rc) {
|
||||
setPackedGitOpenFiles(rc.getInt(
|
||||
"core", null, "packedgitopenfiles", getPackedGitOpenFiles())); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
setPackedGitLimit(rc.getLong(
|
||||
"core", null, "packedgitlimit", getPackedGitLimit())); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
setPackedGitWindowSize(rc.getInt(
|
||||
"core", null, "packedgitwindowsize", getPackedGitWindowSize())); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
setPackedGitMMAP(rc.getBoolean(
|
||||
"core", null, "packedgitmmap", isPackedGitMMAP())); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
setDeltaBaseCacheLimit(rc.getInt(
|
||||
"core", null, "deltabasecachelimit", getDeltaBaseCacheLimit())); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
setPackedGitUseStrongRefs(rc.getBoolean(CONFIG_CORE_SECTION,
|
||||
CONFIG_KEY_PACKED_GIT_USE_STRONGREFS,
|
||||
isPackedGitUseStrongRefs()));
|
||||
setPackedGitOpenFiles(rc.getInt(CONFIG_CORE_SECTION, null,
|
||||
CONFIG_KEY_PACKED_GIT_OPENFILES, getPackedGitOpenFiles()));
|
||||
setPackedGitLimit(rc.getLong(CONFIG_CORE_SECTION, null,
|
||||
CONFIG_KEY_PACKED_GIT_LIMIT, getPackedGitLimit()));
|
||||
setPackedGitWindowSize(rc.getInt(CONFIG_CORE_SECTION, null,
|
||||
CONFIG_KEY_PACKED_GIT_WINDOWSIZE, getPackedGitWindowSize()));
|
||||
setPackedGitMMAP(rc.getBoolean(CONFIG_CORE_SECTION, null,
|
||||
CONFIG_KEY_PACKED_GIT_MMAP, isPackedGitMMAP()));
|
||||
setDeltaBaseCacheLimit(rc.getInt(CONFIG_CORE_SECTION, null,
|
||||
CONFIG_KEY_DELTA_BASE_CACHE_LIMIT, getDeltaBaseCacheLimit()));
|
||||
|
||||
long maxMem = Runtime.getRuntime().maxMemory();
|
||||
long sft = rc.getLong(
|
||||
"core", null, "streamfilethreshold", getStreamFileThreshold()); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
long sft = rc.getLong(CONFIG_CORE_SECTION, null,
|
||||
CONFIG_KEY_STREAM_FILE_TRESHOLD, getStreamFileThreshold());
|
||||
sft = Math.min(sft, maxMem / 4); // don't use more than 1/4 of the heap
|
||||
sft = Math.min(sft, Integer.MAX_VALUE); // cannot exceed array length
|
||||
setStreamFileThreshold((int) sft);
|
||||
|
|
|
@ -42,6 +42,8 @@
|
|||
*/
|
||||
package org.eclipse.jgit.storage.file;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import javax.management.MXBean;
|
||||
|
||||
import org.eclipse.jgit.internal.storage.file.WindowCache;
|
||||
|
@ -225,6 +227,13 @@ default double getAverageLoadTime() {
|
|||
*/
|
||||
long getOpenByteCount();
|
||||
|
||||
/**
|
||||
* Number of bytes cached per repository
|
||||
*
|
||||
* @return number of bytes cached per repository
|
||||
*/
|
||||
Map<String, Long> getOpenByteCountPerRepository();
|
||||
|
||||
/**
|
||||
* Reset counters. Does not reset open bytes and open files counters.
|
||||
*/
|
||||
|
|
|
@ -239,12 +239,10 @@ protected AbstractTreeIterator(AbstractTreeIterator p) {
|
|||
path = p.path;
|
||||
pathOffset = p.pathLen + 1;
|
||||
|
||||
try {
|
||||
path[pathOffset - 1] = '/';
|
||||
} catch (ArrayIndexOutOfBoundsException e) {
|
||||
if (pathOffset > path.length) {
|
||||
growPath(p.pathLen);
|
||||
path[pathOffset - 1] = '/';
|
||||
}
|
||||
path[pathOffset - 1] = '/';
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -387,14 +387,13 @@ private void parseEntry() {
|
|||
tmp = pathOffset;
|
||||
for (;; tmp++) {
|
||||
c = raw[ptr++];
|
||||
if (c == 0)
|
||||
if (c == 0) {
|
||||
break;
|
||||
try {
|
||||
path[tmp] = c;
|
||||
} catch (ArrayIndexOutOfBoundsException e) {
|
||||
growPath(tmp);
|
||||
path[tmp] = c;
|
||||
}
|
||||
if (tmp >= path.length) {
|
||||
growPath(tmp);
|
||||
}
|
||||
path[tmp] = c;
|
||||
}
|
||||
pathLen = tmp;
|
||||
nextPtr = ptr + OBJECT_ID_LENGTH;
|
||||
|
|
Loading…
Reference in New Issue