FileSnapshot: Lazy load file store attributes cache
Doing a getFileStoreAttributes call even when the file doesn't exist is unnecessary. This call is particularly slow on some filesystems. Instead, do it only when the file exists and load the appropriate cache. This update can help speed up RefDirectory.exactRef when the ref is packed, but has a corresponding empty dir for it under 'refs/'. This scenario can happen when an atomic 'BatchRefUpdate' creates new sharded refs. For example, consider the case where we create 50k sharded refs in a new namespace called 'new-refs' using an atomic 'BatchRefUpdate'. The refs are named like 'refs/new-refs/01/1/1', 'refs/new-refs/01/1/2', 'refs/new-refs/01/1/3' and so on. After the refs are created, the 'new-refs' namespace looks like below: $ find refs/new-refs -type f | wc -l 0 $ find refs/new-refs -type d | wc -l 5101 At this point, an 'exactRef' call on each of the 50k refs without this change takes ~30s, where as with this change it takes ~2.5s. Change-Id: I4a5d4c6a652dbeed1f4bc3b4f2b2f1416f7ca0e7 Signed-off-by: Kaushik Lingarkar <quic_kaushikl@quicinc.com>
This commit is contained in:
parent
09c923073a
commit
5606a53151
|
@ -217,6 +217,12 @@ public static FileSnapshot save(Instant modified) {
|
||||||
/** measured FileStore attributes */
|
/** measured FileStore attributes */
|
||||||
private FileStoreAttributes fileStoreAttributeCache;
|
private FileStoreAttributes fileStoreAttributeCache;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* if {@code true} read filesystem time resolution from configuration file
|
||||||
|
* otherwise use fallback resolution
|
||||||
|
*/
|
||||||
|
private boolean useConfig;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Object that uniquely identifies the given file, or {@code
|
* Object that uniquely identifies the given file, or {@code
|
||||||
* null} if a file key is not available
|
* null} if a file key is not available
|
||||||
|
@ -253,9 +259,7 @@ protected FileSnapshot(File file) {
|
||||||
protected FileSnapshot(File file, boolean useConfig) {
|
protected FileSnapshot(File file, boolean useConfig) {
|
||||||
this.file = file;
|
this.file = file;
|
||||||
this.lastRead = Instant.now();
|
this.lastRead = Instant.now();
|
||||||
this.fileStoreAttributeCache = useConfig
|
this.useConfig = useConfig;
|
||||||
? FS.getFileStoreAttributes(file.toPath().getParent())
|
|
||||||
: FALLBACK_FILESTORE_ATTRIBUTES;
|
|
||||||
BasicFileAttributes fileAttributes = null;
|
BasicFileAttributes fileAttributes = null;
|
||||||
try {
|
try {
|
||||||
fileAttributes = FS.DETECTED.fileAttributes(file);
|
fileAttributes = FS.DETECTED.fileAttributes(file);
|
||||||
|
@ -399,7 +403,7 @@ public void setClean(FileSnapshot other) {
|
||||||
* if sleep was interrupted
|
* if sleep was interrupted
|
||||||
*/
|
*/
|
||||||
public void waitUntilNotRacy() throws InterruptedException {
|
public void waitUntilNotRacy() throws InterruptedException {
|
||||||
long timestampResolution = fileStoreAttributeCache
|
long timestampResolution = fileStoreAttributeCache()
|
||||||
.getFsTimestampResolution().toNanos();
|
.getFsTimestampResolution().toNanos();
|
||||||
while (isRacyClean(Instant.now())) {
|
while (isRacyClean(Instant.now())) {
|
||||||
TimeUnit.NANOSECONDS.sleep(timestampResolution);
|
TimeUnit.NANOSECONDS.sleep(timestampResolution);
|
||||||
|
@ -519,10 +523,10 @@ private boolean isRacyClean(Instant read) {
|
||||||
}
|
}
|
||||||
|
|
||||||
private long getEffectiveRacyThreshold() {
|
private long getEffectiveRacyThreshold() {
|
||||||
long timestampResolution = fileStoreAttributeCache
|
long timestampResolution = fileStoreAttributeCache()
|
||||||
.getFsTimestampResolution().toNanos();
|
.getFsTimestampResolution().toNanos();
|
||||||
long minRacyInterval = fileStoreAttributeCache.getMinimalRacyInterval()
|
long minRacyInterval = fileStoreAttributeCache()
|
||||||
.toNanos();
|
.getMinimalRacyInterval().toNanos();
|
||||||
long max = Math.max(timestampResolution, minRacyInterval);
|
long max = Math.max(timestampResolution, minRacyInterval);
|
||||||
// safety margin: factor 2.5 below 100ms otherwise 1.25
|
// safety margin: factor 2.5 below 100ms otherwise 1.25
|
||||||
return max < 100_000_000L ? max * 5 / 2 : max * 5 / 4;
|
return max < 100_000_000L ? max * 5 / 2 : max * 5 / 4;
|
||||||
|
@ -582,4 +586,13 @@ private boolean isSizeChanged(long currSize) {
|
||||||
}
|
}
|
||||||
return changed;
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private FileStoreAttributes fileStoreAttributeCache() {
|
||||||
|
if (fileStoreAttributeCache == null) {
|
||||||
|
fileStoreAttributeCache = useConfig
|
||||||
|
? FS.getFileStoreAttributes(file.toPath().getParent())
|
||||||
|
: FALLBACK_FILESTORE_ATTRIBUTES;
|
||||||
|
}
|
||||||
|
return fileStoreAttributeCache;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue