motiejus
/
jgit
1
Fork 0
Code Releases Activity

Rescale "Compressing objects" progress meter by size 

Instead of counting objects processed, count number of bytes added
into the window. This should rescale the progress meter so that 30%
complete means 30% of the total uncompressed content size has been
inflated and fed into the window.

In theory the progress meter should be more accurate about its
percentage complete/remaining fraction than with objects. When
counting objects small objects move the progress meter more rapidly
than large objects, but demand a smaller amount of work than large
objects being compressed.

Change-Id: Id2848c16a2148b5ca51e0ca1e29c5be97eefeb48
This commit is contained in:
Shawn Pearce 2013-04-16 09:25:29 -07:00 committed by Gerrit Code Review @ Eclipse.org
parent 21e4aa2b9e
commit 5d8a9f6f3f
3 changed files with 58 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaTask.java
View File

@ -57,6 +57,8 @@
import org.eclipse.jgit.storage.pack.PackConfig;
final class DeltaTask implements Callable<Object> {
static final long MAX_METER = 9 << 20;
static final class Block {
private static final int MIN_TOP_PATH = 50 << 20;
@ -71,6 +73,7 @@ static final class Block {
final int endIndex;
private long totalWeight;
private long bytesPerUnit;
Block(int threads, PackConfig config, ObjectReader reader,
DeltaCache dc, ThreadSafeProgressMonitor pm,
@ -86,6 +89,13 @@ static final class Block {
this.endIndex = end;
}
int cost() {
int d = (int) (totalWeight / bytesPerUnit);
if (totalWeight % bytesPerUnit != 0)
d++;
return d;
}
synchronized DeltaWindow stealWork(DeltaTask forThread) {
for (;;) {
DeltaTask maxTask = null;
@ -200,6 +210,10 @@ public int compare(WeightedPath a, WeightedPath b) {
return a.slice.beginIndex - b.slice.beginIndex;
}
});
bytesPerUnit = 1;
while (MAX_METER <= (totalWeight / bytesPerUnit))
bytesPerUnit <<= 10;
return topPaths;
}
}
@ -282,7 +296,7 @@ public Object call() throws Exception {
DeltaWindow initWindow(Slice s) {
DeltaWindow w = new DeltaWindow(block.config, block.dc,
or, block.pm,
or, block.pm, block.bytesPerUnit,
block.list, s.beginIndex, s.endIndex);
synchronized (this) {
dw = w;

13
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/DeltaWindow.java
View File

@ -64,6 +64,8 @@ final class DeltaWindow {
private final DeltaCache deltaCache;
private final ObjectReader reader;
private final ProgressMonitor monitor;
private final long bytesPerUnit;
private long bytesProcessed;
/** Maximum number of bytes to admit to the window at once. */
private final long maxMemory;
@ -92,12 +94,13 @@ final class DeltaWindow {
private Deflater deflater;
DeltaWindow(PackConfig pc, DeltaCache dc, ObjectReader or,
ProgressMonitor pm,
ProgressMonitor pm, long bpu,
ObjectToPack[] in, int beginIndex, int endIndex) {
config = pc;
deltaCache = dc;
reader = or;
monitor = pm;
bytesPerUnit = bpu;
toSearch = in;
cur = beginIndex;
end = endIndex;
@ -162,12 +165,14 @@ void search() throws IOException {
// We don't actually want to make a delta for
// them, just need to push them into the window
// so they can be read by other objects.
//
keepInWindow();
} else {
// Search for a delta for the current window slot.
//
monitor.update(1);
if (bytesPerUnit <= (bytesProcessed += next.getWeight())) {
int d = (int) (bytesProcessed / bytesPerUnit);
monitor.update(d);
bytesProcessed -= d * bytesPerUnit;
}
searchInWindow();
}
}

45
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java
View File

@ -1286,9 +1286,7 @@ public int compare(ObjectToPack a, ObjectToPack b) {
return;
final long searchStart = System.currentTimeMillis();
beginPhase(PackingPhase.COMPRESSING, monitor, nonEdgeCnt);
searchForDeltas(monitor, list, cnt);
endPhase(monitor);
stats.deltaSearchNonEdgeObjects = nonEdgeCnt;
stats.timeCompressing = System.currentTimeMillis() - searchStart;
@ -1327,25 +1325,49 @@ private void searchForDeltas(final ProgressMonitor monitor,
int threads = config.getThreads();
if (threads == 0)
threads = Runtime.getRuntime().availableProcessors();
if (threads <= 1 || cnt <= config.getDeltaSearchWindowSize())
singleThreadDeltaSearch(monitor, list, cnt);
else
parallelDeltaSearch(monitor, list, cnt, threads);
}
if (threads <= 1 || cnt <= 2 * config.getDeltaSearchWindowSize()) {
new DeltaWindow(config, new DeltaCache(config), reader, monitor,
list, 0, cnt).search();
return;
private void singleThreadDeltaSearch(ProgressMonitor monitor,
ObjectToPack[] list, int cnt) throws IOException {
long totalWeight = 0;
for (int i = 0; i < cnt; i++) {
ObjectToPack o = list[i];
if (!o.isEdge() && !o.doNotAttemptDelta())
totalWeight += o.getWeight();
}
final DeltaCache dc = new ThreadSafeDeltaCache(config);
final ThreadSafeProgressMonitor pm = new ThreadSafeProgressMonitor(monitor);
long bytesPerUnit = 1;
while (DeltaTask.MAX_METER <= (totalWeight / bytesPerUnit))
bytesPerUnit <<= 10;
int cost = (int) (totalWeight / bytesPerUnit);
if (totalWeight % bytesPerUnit != 0)
cost++;
beginPhase(PackingPhase.COMPRESSING, monitor, cost);
new DeltaWindow(config, new DeltaCache(config), reader,
monitor, bytesPerUnit,
list, 0, cnt).search();
endPhase(monitor);
}
private void parallelDeltaSearch(ProgressMonitor monitor,
ObjectToPack[] list, int cnt, int threads) throws IOException {
DeltaCache dc = new ThreadSafeDeltaCache(config);
ThreadSafeProgressMonitor pm = new ThreadSafeProgressMonitor(monitor);
DeltaTask.Block taskBlock = new DeltaTask.Block(threads, config,
reader, dc, pm,
list, 0, cnt);
taskBlock.partitionTasks();
beginPhase(PackingPhase.COMPRESSING, monitor, taskBlock.cost());
pm.startWorkers(taskBlock.tasks.size());
final Executor executor = config.getExecutor();
final List<Throwable> errors = Collections
.synchronizedList(new ArrayList<Throwable>());
Executor executor = config.getExecutor();
final List<Throwable> errors =
Collections.synchronizedList(new ArrayList<Throwable>(threads));
if (executor instanceof ExecutorService) {
// Caller supplied us a service, use it directly.
runTasks((ExecutorService) executor, pm, taskBlock, errors);
@ -1409,6 +1431,7 @@ public void run() {
fail.initCause(err);
throw fail;
}
endPhase(monitor);
}
private static void runTasks(ExecutorService pool,