motiejus
/
jgit
1
Fork 0
Code Releases Activity

Use limited getCachedBytes in RevWalk 

Parsing is rewritten to use the size limited form of getCachedBytes,
thus freeing the revwalk infrastructure from needing to care about
a large object vs. a small object when it gets an ObjectLoader.

Right now we hardcode our upper bound for a commit or annotated
tag to be 15 MiB.  I don't know of any that is more than 1 MiB in
the wild, so going 15x that should give us some reasonable headroom.

Change-Id: If296c211d8b257d76e44908504e71dd9ba70ffa8
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
This commit is contained in:
Shawn O. Pearce 2010-08-24 12:50:59 -07:00
parent c11711f98e
commit 127a5f95e1
6 changed files with 78 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevBlob.java
View File

@ -45,6 +45,10 @@
package org.eclipse.jgit.revwalk;
import java.io.IOException;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
@ -64,4 +68,20 @@ protected RevBlob(final AnyObjectId id) {
public final int getType() {
return Constants.OBJ_BLOB;
}
@Override
void parseHeaders(RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if (walk.reader.has(this))
flags |= PARSED;
else
throw new MissingObjectException(this, getType());
}
@Override
void parseBody(RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if ((flags & PARSED) == 0)
parseHeaders(walk);
}
}

4
org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java
View File

@ -133,14 +133,14 @@ protected RevCommit(final AnyObjectId id) {
@Override
void parseHeaders(final RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
parseCanonical(walk, loadCanonical(walk));
parseCanonical(walk, walk.getCachedBytes(this));
}
@Override
void parseBody(final RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if (buffer == null) {
buffer = loadCanonical(walk);
buffer = walk.getCachedBytes(this);
if ((flags & PARSED) == 0)
parseCanonical(walk, buffer);
}

45
org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevObject.java
View File

@ -45,64 +45,27 @@
import java.io.IOException;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectStream;
import org.eclipse.jgit.util.IO;
/** Base object type accessed during revision walking. */
public abstract class RevObject extends ObjectId {
static final int PARSED = 1;
static byte[] asByteArray(RevObject obj, ObjectLoader loader) throws IOException {
if (loader.isLarge()) {
ObjectStream in = loader.openStream();
try {
long sz = in.getSize();
if (Integer.MAX_VALUE <= sz){
if (obj != null)
throw new LargeObjectException(obj.copy());
throw new LargeObjectException();
}
byte[] buf = new byte[(int) sz];
IO.readFully(in, buf, 0, buf.length);
return buf;
} finally {
in.close();
}
}
return loader.getCachedBytes();
}
int flags;
RevObject(final AnyObjectId name) {
super(name);
}
void parseHeaders(final RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
loadCanonical(walk);
flags |= PARSED;
}
abstract void parseHeaders(RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException;
void parseBody(final RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if ((flags & PARSED) == 0)
parseHeaders(walk);
}
final byte[] loadCanonical(final RevWalk walk) throws IOException,
MissingObjectException, IncorrectObjectTypeException,
CorruptObjectException {
return asByteArray(this, walk.reader.open(this, getType()));
}
abstract void parseBody(RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException;
/**
* Get Git object type. See {@link Constants}.

4
org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java
View File

@ -131,14 +131,14 @@ protected RevTag(final AnyObjectId id) {
@Override
void parseHeaders(final RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
parseCanonical(walk, loadCanonical(walk));
parseCanonical(walk, walk.getCachedBytes(this));
}
@Override
void parseBody(final RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if (buffer == null) {
buffer = loadCanonical(walk);
buffer = walk.getCachedBytes(this);
if ((flags & PARSED) == 0)
parseCanonical(walk, buffer);
}

20
org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTree.java
View File

@ -45,6 +45,10 @@
package org.eclipse.jgit.revwalk;
import java.io.IOException;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
@ -64,4 +68,20 @@ protected RevTree(final AnyObjectId id) {
public final int getType() {
return Constants.OBJ_TREE;
}
@Override
void parseHeaders(RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if (walk.reader.has(this))
flags |= PARSED;
else
throw new MissingObjectException(this, getType());
}
@Override
void parseBody(RevWalk walk) throws MissingObjectException,
IncorrectObjectTypeException, IOException {
if ((flags & PARSED) == 0)
parseHeaders(walk);
}
}

33
org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java
View File

@ -61,6 +61,7 @@
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.CoreConfig;
import org.eclipse.jgit.lib.MutableObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdSubclassMap;
@ -170,6 +171,9 @@ public class RevWalk implements Iterable<RevCommit> {
private final ObjectIdSubclassMap<RevObject> objects;
/** Largest commit or annotated tag we are willing to touch. */
private final int bigFileThreshold;
private int freeFlags = APP_FLAGS;
private int delayFreeFlags;
@ -226,6 +230,13 @@ private RevWalk(final Repository repo, final ObjectReader or) {
filter = RevFilter.ALL;
treeFilter = TreeFilter.ALL;
retainBody = true;
if (repo != null) {
CoreConfig cfg = repo.getConfig().get(CoreConfig.KEY);
bigFileThreshold = cfg.getStreamFileThreshold();
} else {
bigFileThreshold = 15 * 1024 * 1024;
}
}
/** @return the reader this walker is using to load objects. */
@ -813,13 +824,14 @@ public RevObject parseAny(final AnyObjectId id)
}
private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
throws CorruptObjectException, LargeObjectException {
throws LargeObjectException, CorruptObjectException,
MissingObjectException, IOException {
RevObject r;
int type = ldr.getType();
switch (type) {
case Constants.OBJ_COMMIT: {
final RevCommit c = createCommit(id);
c.parseCanonical(this, ldr.getCachedBytes());
c.parseCanonical(this, getCachedBytes(c, ldr));
r = c;
break;
}
@ -835,7 +847,7 @@ private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
}
case Constants.OBJ_TAG: {
final RevTag t = new RevTag(id);
t.parseCanonical(this, ldr.getCachedBytes());
t.parseCanonical(this, getCachedBytes(t, ldr));
r = t;
break;
}
@ -847,6 +859,21 @@ private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
return r;
}
byte[] getCachedBytes(RevObject obj) throws LargeObjectException,
MissingObjectException, IncorrectObjectTypeException, IOException {
return getCachedBytes(obj, reader.open(obj, obj.getType()));
}
byte[] getCachedBytes(RevObject obj, ObjectLoader ldr)
throws LargeObjectException, MissingObjectException, IOException {
try {
return ldr.getCachedBytes(bigFileThreshold);
} catch (LargeObjectException tooBig) {
tooBig.setObjectId(obj);
throw tooBig;
}
}
/**
* Asynchronous object parsing.
*