motiejus
/
jgit
1
Fork 0
Code Releases Activity

Merge changes I2645d482,Ic81fefb1,Id64ab38d 

* changes:
  Remove cached_packs support in favor of bitmaps
  Remove objects before optimization from DfsGarbageCollector
  Simplfy caching of DfsPackDescription from PackWriter.Statistics
This commit is contained in:
Shawn Pearce 2013-03-18 10:35:31 -04:00 committed by Gerrit Code Review @ Eclipse.org
parent 44ea46dd40 3760e4319b
commit 462bbc052e
13 changed files with 22 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsCachedPack.java
View File

@ -44,9 +44,7 @@
package org.eclipse.jgit.storage.dfs;
import java.io.IOException;
import java.util.Set;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.storage.pack.CachedPack;
import org.eclipse.jgit.storage.pack.ObjectToPack;
import org.eclipse.jgit.storage.pack.PackOutputStream;
@ -65,11 +63,6 @@ public DfsPackDescription getPackDescription() {
return pack.getPackDescription();
}
@Override
public Set<ObjectId> getTips() {
return getPackDescription().getTips();
}
@Override
public long getObjectCount() throws IOException {
return getPackDescription().getObjectCount();

30
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsGarbageCollector.java
View File

@ -100,12 +100,6 @@ public class DfsGarbageCollector {
private Set<ObjectId> nonHeads;
/** Sum of object counts in {@link #packsBefore}. */
private long objectsBefore;
/** Sum of object counts iN {@link #newPackDesc}. */
private long objectsPacked;
private Set<ObjectId> tagTargets;
/**
@ -281,14 +275,14 @@ private void packHeads(ProgressMonitor pm) throws IOException {
try {
pw.preparePack(pm, allHeads, Collections.<ObjectId> emptySet());
if (0 < pw.getObjectCount())
writePack(GC, pw, pm).setTips(allHeads);
writePack(GC, pw, pm);
} finally {
pw.release();
}
}
private void packRest(ProgressMonitor pm) throws IOException {
if (nonHeads.isEmpty() || objectsPacked == getObjectsBefore())
if (nonHeads.isEmpty())
return;
PackWriter pw = newPackWriter();
@ -304,14 +298,11 @@ private void packRest(ProgressMonitor pm) throws IOException {
}
private void packGarbage(ProgressMonitor pm) throws IOException {
if (objectsPacked == getObjectsBefore())
return;
// TODO(sop) This is ugly. The garbage pack needs to be deleted.
PackWriter pw = newPackWriter();
try {
RevWalk pool = new RevWalk(ctx);
pm.beginTask("Finding garbage", (int) getObjectsBefore());
pm.beginTask("Finding garbage", objectsBefore());
for (DfsPackFile oldPack : packsBefore) {
PackIndex oldIdx = oldPack.getPackIndex(ctx);
for (PackIndex.MutableEntry ent : oldIdx) {
@ -343,12 +334,11 @@ private static boolean isHead(Ref ref) {
return ref.getName().startsWith(Constants.R_HEADS);
}
private long getObjectsBefore() {
if (objectsBefore == 0) {
for (DfsPackFile p : packsBefore)
objectsBefore += p.getPackDescription().getObjectCount();
}
return objectsBefore;
private int objectsBefore() {
int cnt = 0;
for (DfsPackFile p : packsBefore)
cnt += p.getPackDescription().getObjectCount();
return cnt;
}
private PackWriter newPackWriter() {
@ -403,10 +393,6 @@ public boolean contains(AnyObjectId objectId) {
PackWriter.Statistics stats = pw.getStatistics();
pack.setPackStats(stats);
pack.setFileSize(PACK, stats.getTotalBytes());
pack.setObjectCount(stats.getTotalObjects());
pack.setDeltaCount(stats.getTotalDeltas());
objectsPacked += stats.getTotalObjects();
newPackStats.add(stats);
DfsBlockCache.getInstance().getOrCreate(pack, null);

11
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackCompactor.java
View File

@ -283,20 +283,19 @@ public int compare(ObjectIdWithOffset a, ObjectIdWithOffset b) {
pm.endTask();
}
private void writePack(DfsObjDatabase objdb, DfsPackDescription pack,
private static void writePack(DfsObjDatabase objdb,
DfsPackDescription pack,
PackWriter pw, ProgressMonitor pm) throws IOException {
DfsOutputStream out = objdb.writeFile(pack, PACK);
try {
CountingOutputStream cnt = new CountingOutputStream(out);
pw.writePack(pm, pm, cnt);
pack.setObjectCount(pw.getObjectCount());
pack.setFileSize(PACK, cnt.getCount());
pw.writePack(pm, pm, out);
} finally {
out.close();
}
}
private void writeIndex(DfsObjDatabase objdb, DfsPackDescription pack,
private static void writeIndex(DfsObjDatabase objdb,
DfsPackDescription pack,
PackWriter pw) throws IOException {
DfsOutputStream out = objdb.writeFile(pack, INDEX);
try {

24
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsPackDescription.java
View File

@ -43,11 +43,11 @@
package org.eclipse.jgit.storage.dfs;
import static org.eclipse.jgit.storage.pack.PackExt.PACK;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.storage.dfs.DfsObjDatabase.PackSource;
import org.eclipse.jgit.storage.pack.PackExt;
import org.eclipse.jgit.storage.pack.PackWriter;
@ -75,8 +75,6 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> {
private long deltaCount;
private Set<ObjectId> tips;
private PackWriter.Statistics stats;
private int extensions;
@ -221,21 +219,6 @@ public DfsPackDescription setDeltaCount(long cnt) {
return this;
}
/** @return the tips that created this pack, if known. */
public Set<ObjectId> getTips() {
return tips;
}
/**
* @param tips
* the tips of the pack, null if it has no known tips.
* @return {@code this}
*/
public DfsPackDescription setTips(Set<ObjectId> tips) {
this.tips = tips;
return this;
}
/**
* @return statistics from PackWriter, if the pack was built with it.
* Generally this is only available for packs created by
@ -248,6 +231,9 @@ public PackWriter.Statistics getPackStats() {
DfsPackDescription setPackStats(PackWriter.Statistics stats) {
this.stats = stats;
setFileSize(PACK, stats.getTotalBytes());
setObjectCount(stats.getTotalObjects());
setDeltaCount(stats.getTotalDeltas());
return this;
}

15
org.eclipse.jgit/src/org/eclipse/jgit/storage/dfs/DfsReader.java
View File

@ -593,21 +593,6 @@ public void writeObjects(PackOutputStream out, List<ObjectToPack> list)
}
}
public Collection<CachedPack> getCachedPacks() throws IOException {
DfsPackFile[] packList = db.getPacks();
List<CachedPack> cached = new ArrayList<CachedPack>(packList.length);
for (DfsPackFile pack : packList) {
DfsPackDescription desc = pack.getPackDescription();
if (canBeCachedPack(desc))
cached.add(new DfsCachedPack(pack));
}
return cached;
}
private static boolean canBeCachedPack(DfsPackDescription desc) {
return desc.getTips() != null && !desc.getTips().isEmpty();
}
public void copyPackAsIs(PackOutputStream out, CachedPack pack,
boolean validate) throws IOException {
try {

6
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java
View File

@ -57,7 +57,6 @@
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdOwnerMap;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.storage.pack.CachedPack;
import org.eclipse.jgit.storage.pack.ObjectToPack;
import org.eclipse.jgit.storage.pack.PackWriter;
import org.eclipse.jgit.util.FS;
@ -146,11 +145,6 @@ Set<ObjectId> getShallowCommits() throws IOException {
return wrapped.getShallowCommits();
}
@Override
Collection<? extends CachedPack> getCachedPacks() throws IOException {
return wrapped.getCachedPacks();
}
@Override
AlternateHandle[] myAlternates() {
if (alts == null) {

9
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java
View File

@ -55,7 +55,6 @@
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.storage.pack.CachedPack;
import org.eclipse.jgit.storage.pack.ObjectToPack;
import org.eclipse.jgit.storage.pack.PackWriter;
import org.eclipse.jgit.util.FS;
@ -262,9 +261,6 @@ abstract void selectObjectRepresentation(PackWriter packer,
abstract File getDirectory();
abstract Collection<? extends CachedPack> getCachedPacks()
throws IOException;
abstract AlternateHandle[] myAlternates();
abstract boolean tryAgain1();
@ -301,11 +297,6 @@ static class AlternateHandle {
this.db = db;
}
@SuppressWarnings("unchecked")
Collection<CachedPack> getCachedPacks() throws IOException {
return (Collection<CachedPack>) db.getCachedPacks();
}
void close() {
db.close();
}

20
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/LocalCachedPack.java
View File

@ -46,11 +46,8 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.storage.pack.CachedPack;
import org.eclipse.jgit.storage.pack.ObjectToPack;
import org.eclipse.jgit.storage.pack.PackOutputStream;
@ -59,36 +56,21 @@
class LocalCachedPack extends CachedPack {
private final ObjectDirectory odb;
private final Set<ObjectId> tips;
private final String[] packNames;
private PackFile[] packs;
LocalCachedPack(ObjectDirectory odb, Set<ObjectId> tips,
List<String> packNames) {
LocalCachedPack(ObjectDirectory odb, List<String> packNames) {
this.odb = odb;
if (tips.size() == 1)
this.tips = Collections.singleton(tips.iterator().next());
else
this.tips = Collections.unmodifiableSet(tips);
this.packNames = packNames.toArray(new String[packNames.size()]);
}
LocalCachedPack(List<PackFile> packs) {
odb = null;
tips = null;
packNames = null;
this.packs = packs.toArray(new PackFile[packs.size()]);
}
@Override
public Set<ObjectId> getTips() {
return tips;
}
@Override
public long getObjectCount() throws IOException {
long cnt = 0;

106
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java
View File

@ -75,14 +75,11 @@
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.RepositoryCache;
import org.eclipse.jgit.lib.RepositoryCache.FileKey;
import org.eclipse.jgit.storage.pack.CachedPack;
import org.eclipse.jgit.storage.pack.ObjectToPack;
import org.eclipse.jgit.storage.pack.PackExt;
import org.eclipse.jgit.storage.pack.PackWriter;
import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.FileUtils;
import org.eclipse.jgit.util.IO;
import org.eclipse.jgit.util.RawParseUtils;
/**
* Traditional file system based {@link ObjectDatabase}.
@ -119,12 +116,8 @@ public class ObjectDirectory extends FileObjectDatabase {
private final File alternatesFile;
private final File cachedPacksFile;
private final AtomicReference<PackList> packList;
private final AtomicReference<CachedPackList> cachedPacks;
private final FS fs;
private final AtomicReference<AlternateHandle[]> alternates;
@ -162,9 +155,7 @@ public ObjectDirectory(final Config cfg, final File dir,
infoDirectory = new File(objects, "info"); //$NON-NLS-1$
packDirectory = new File(objects, "pack"); //$NON-NLS-1$
alternatesFile = new File(infoDirectory, "alternates"); //$NON-NLS-1$
cachedPacksFile = new File(infoDirectory, "cached-packs"); //$NON-NLS-1$
packList = new AtomicReference<PackList>(NO_PACKS);
cachedPacks = new AtomicReference<CachedPackList>();
unpackedObjectCache = new UnpackedObjectCache();
this.fs = fs;
this.shallowFile = shallowFile;
@ -250,83 +241,6 @@ public Collection<PackFile> getPacks() {
return Collections.unmodifiableCollection(Arrays.asList(packs));
}
@Override
Collection<? extends CachedPack> getCachedPacks() throws IOException {
CachedPackList list = cachedPacks.get();
if (list == null || list.snapshot.isModified(cachedPacksFile))
list = scanCachedPacks(list);
Collection<CachedPack> result = list.getCachedPacks();
boolean resultIsCopy = false;
for (AlternateHandle h : myAlternates()) {
Collection<CachedPack> altPacks = h.getCachedPacks();
if (altPacks.isEmpty())
continue;
if (result.isEmpty()) {
result = altPacks;
continue;
}
if (!resultIsCopy) {
result = new ArrayList<CachedPack>(result);
resultIsCopy = true;
}
result.addAll(altPacks);
}
return result;
}
private CachedPackList scanCachedPacks(CachedPackList old)
throws IOException {
FileSnapshot s = FileSnapshot.save(cachedPacksFile);
byte[] buf;
try {
buf = IO.readFully(cachedPacksFile);
} catch (FileNotFoundException e) {
buf = new byte[0];
}
if (old != null && old.snapshot.equals(s)
&& Arrays.equals(old.raw, buf)) {
old.snapshot.setClean(s);
return old;
}
ArrayList<LocalCachedPack> list = new ArrayList<LocalCachedPack>(4);
Set<ObjectId> tips = new HashSet<ObjectId>();
int ptr = 0;
while (ptr < buf.length) {
if (buf[ptr] == '#' || buf[ptr] == '\n') {
ptr = RawParseUtils.nextLF(buf, ptr);
continue;
}
if (buf[ptr] == '+') {
tips.add(ObjectId.fromString(buf, ptr + 2));
ptr = RawParseUtils.nextLF(buf, ptr + 2);
continue;
}
List<String> names = new ArrayList<String>(4);
while (ptr < buf.length && buf[ptr] == 'P') {
int end = RawParseUtils.nextLF(buf, ptr);
if (buf[end - 1] == '\n')
end--;
names.add(RawParseUtils.decode(buf, ptr + 2, end));
ptr = RawParseUtils.nextLF(buf, end);
}
if (!tips.isEmpty() && !names.isEmpty()) {
list.add(new LocalCachedPack(this, tips, names));
tips = new HashSet<ObjectId>();
}
}
list.trimToSize();
return new CachedPackList(s, Collections.unmodifiableList(list), buf);
}
/**
* Add a single existing pack to the list of available pack files.
*
@ -893,26 +807,6 @@ private static final class PackList {
}
}
private static final class CachedPackList {
final FileSnapshot snapshot;
final Collection<LocalCachedPack> packs;
final byte[] raw;
CachedPackList(FileSnapshot sn, List<LocalCachedPack> list, byte[] buf) {
snapshot = sn;
packs = list;
raw = buf;
}
@SuppressWarnings("unchecked")
Collection<CachedPack> getCachedPacks() {
Collection p = packs;
return p;
}
}
@Override
public ObjectDatabase newCachedDatabase() {
return newCachedFileObjectDatabase();

5
org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java
View File

@ -196,11 +196,6 @@ public void writeObjects(PackOutputStream out, List<ObjectToPack> list)
out.writeObject(otp);
}
@SuppressWarnings("unchecked")
public Collection<CachedPack> getCachedPacks() throws IOException {
return (Collection<CachedPack>) db.getCachedPacks();
}
/**
* Copy bytes from the window to a caller supplied buffer.
*

15
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/CachedPack.java
View File

@ -44,24 +44,9 @@
package org.eclipse.jgit.storage.pack;
import java.io.IOException;
import java.util.Set;
import org.eclipse.jgit.lib.ObjectId;
/** Describes a pack file {@link ObjectReuseAsIs} can append onto a stream. */
public abstract class CachedPack {
/**
* Objects that start this pack.
* <p>
* All objects reachable from the tips are contained within this pack. If
* {@link PackWriter} is going to include everything reachable from all of
* these objects, this cached pack is eligible to be appended directly onto
* the output pack stream.
*
* @return the tip objects that describe this pack.
*/
public abstract Set<ObjectId> getTips();
/**
* Get the number of objects in this pack.
*

16
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java
View File

@ -89,7 +89,7 @@ public interface ObjectReuseAsIs {
* the writer can select the most suitable representation to reuse into the
* output stream.
* <p>
* If the implementation returns CachedPack from {@link #getCachedPacks()},
* If the implementation returns CachedPack from {@link #getCachedPacksAndUpdate(BitmapBuilder)}
* it must consider the representation of any object that is stored in any
* of the offered CachedPacks. PackWriter relies on this behavior to prune
* duplicate objects out of the pack stream when it selects a CachedPack and
@ -200,20 +200,6 @@ public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp,
boolean validate) throws IOException,
StoredObjectRepresentationNotAvailableException;
/**
* Obtain the available cached packs.
* <p>
* A cached pack has known starting points and may be sent entirely as-is,
* with almost no effort on the sender's part.
*
* @return the available cached packs.
* @throws IOException
* the cached packs cannot be listed from the repository.
* Callers may choose to ignore this and continue as-if there
* were no cached packs.
*/
public Collection<CachedPack> getCachedPacks() throws IOException;
/**
* Append an entire pack's contents onto the output stream.
* <p>

84
org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java
View File

@ -57,10 +57,8 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@ -101,7 +99,6 @@
import org.eclipse.jgit.revwalk.ObjectWalk;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevFlag;
import org.eclipse.jgit.revwalk.RevFlagSet;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevSort;
import org.eclipse.jgit.revwalk.RevTag;
@ -1387,7 +1384,8 @@ public void run() {
}
}
private void runTasks(ExecutorService pool, ThreadSafeProgressMonitor pm,
private static void runTasks(ExecutorService pool,
ThreadSafeProgressMonitor pm,
List<DeltaTask> tasks, List<Throwable> errors) throws IOException {
List<Future<?>> futures = new ArrayList<Future<?>>(tasks.size());
for (DeltaTask task : tasks)
@ -1631,46 +1629,14 @@ private void findObjectsToPack(final ProgressMonitor countingMonitor,
all.addAll(want);
all.addAll(have);
final Map<ObjectId, CachedPack> tipToPack = new HashMap<ObjectId, CachedPack>();
final RevFlag inCachedPack = walker.newFlag("inCachedPack"); //$NON-NLS-1$
final RevFlag include = walker.newFlag("include"); //$NON-NLS-1$
final RevFlag added = walker.newFlag("added"); //$NON-NLS-1$
final RevFlagSet keepOnRestart = new RevFlagSet();
keepOnRestart.add(inCachedPack);
walker.carry(include);
int haveEst = have.size();
if (have.isEmpty()) {
walker.sort(RevSort.COMMIT_TIME_DESC);
if (useCachedPacks && reuseSupport != null) {
Set<ObjectId> need = new HashSet<ObjectId>(want);
List<CachedPack> shortCircuit = new LinkedList<CachedPack>();
for (CachedPack pack : reuseSupport.getCachedPacks()) {
if (need.containsAll(pack.getTips())) {
need.removeAll(pack.getTips());
shortCircuit.add(pack);
}
for (ObjectId id : pack.getTips()) {
tipToPack.put(id, pack);
all.add(id);
}
}
if (need.isEmpty() && !shortCircuit.isEmpty()) {
cachedPacks.addAll(shortCircuit);
for (CachedPack pack : shortCircuit)
countingMonitor.update((int) pack.getObjectCount());
endPhase(countingMonitor);
stats.timeCounting = System.currentTimeMillis() - countingStart;
return;
}
haveEst += tipToPack.size();
}
} else {
walker.sort(RevSort.TOPO);
if (thin)
@ -1688,10 +1654,6 @@ private void findObjectsToPack(final ProgressMonitor countingMonitor,
RevObject o = q.next();
if (o == null)
break;
if (tipToPack.containsKey(o))
o.add(inCachedPack);
if (have.contains(o))
haveObjs.add(o);
if (want.contains(o)) {
@ -1747,20 +1709,6 @@ private void findObjectsToPack(final ProgressMonitor countingMonitor,
while ((c = walker.next()) != null) {
if (exclude(c))
continue;
if (c.has(inCachedPack)) {
CachedPack pack = tipToPack.get(c);
if (includesAllTips(pack, include, walker)) {
useCachedPack(walker, keepOnRestart, //
wantObjs, haveObjs, pack);
commits = new BlockList<RevCommit>();
endPhase(countingMonitor);
beginPhase(PackingPhase.COUNTING, countingMonitor,
ProgressMonitor.UNKNOWN);
continue;
}
}
if (c.has(RevFlag.UNINTERESTING)) {
if (baseTrees.size() <= maxBases)
baseTrees.add(c.getTree());
@ -1893,34 +1841,6 @@ private static void pruneEdgesFromObjectList(List<ObjectToPack> list) {
list.remove(list.size() - 1);
}
private void useCachedPack(ObjectWalk walker, RevFlagSet keepOnRestart,
List<RevObject> wantObj, List<RevObject> baseObj, CachedPack pack)
throws MissingObjectException, IncorrectObjectTypeException,
IOException {
cachedPacks.add(pack);
for (ObjectId id : pack.getTips())
baseObj.add(walker.lookupOrNull(id));
setThin(true);
walker.resetRetain(keepOnRestart);
walker.sort(RevSort.TOPO);
walker.sort(RevSort.BOUNDARY, true);
for (RevObject id : wantObj)
walker.markStart(id);
for (RevObject id : baseObj)
walker.markUninteresting(id);
}
private static boolean includesAllTips(CachedPack pack, RevFlag include,
ObjectWalk walker) {
for (ObjectId id : pack.getTips()) {
if (!walker.lookupOrNull(id).has(include))
return false;
}
return true;
}
/**
* Include one object to the output file.
* <p>