Use limited getCachedBytes in RevWalk
Parsing is rewritten to use the size limited form of getCachedBytes, thus freeing the revwalk infrastructure from needing to care about a large object vs. a small object when it gets an ObjectLoader. Right now we hardcode our upper bound for a commit or annotated tag to be 15 MiB. I don't know of any that is more than 1 MiB in the wild, so going 15x that should give us some reasonable headroom. Change-Id: If296c211d8b257d76e44908504e71dd9ba70ffa8 Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
This commit is contained in:
parent
c11711f98e
commit
127a5f95e1
|
@ -45,6 +45,10 @@
|
|||
|
||||
package org.eclipse.jgit.revwalk;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||
import org.eclipse.jgit.errors.MissingObjectException;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
|
||||
|
@ -64,4 +68,20 @@ protected RevBlob(final AnyObjectId id) {
|
|||
public final int getType() {
|
||||
return Constants.OBJ_BLOB;
|
||||
}
|
||||
|
||||
@Override
|
||||
void parseHeaders(RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if (walk.reader.has(this))
|
||||
flags |= PARSED;
|
||||
else
|
||||
throw new MissingObjectException(this, getType());
|
||||
}
|
||||
|
||||
@Override
|
||||
void parseBody(RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if ((flags & PARSED) == 0)
|
||||
parseHeaders(walk);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -133,14 +133,14 @@ protected RevCommit(final AnyObjectId id) {
|
|||
@Override
|
||||
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
parseCanonical(walk, loadCanonical(walk));
|
||||
parseCanonical(walk, walk.getCachedBytes(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
void parseBody(final RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if (buffer == null) {
|
||||
buffer = loadCanonical(walk);
|
||||
buffer = walk.getCachedBytes(this);
|
||||
if ((flags & PARSED) == 0)
|
||||
parseCanonical(walk, buffer);
|
||||
}
|
||||
|
|
|
@ -45,64 +45,27 @@
|
|||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.eclipse.jgit.errors.CorruptObjectException;
|
||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||
import org.eclipse.jgit.errors.LargeObjectException;
|
||||
import org.eclipse.jgit.errors.MissingObjectException;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.ObjectLoader;
|
||||
import org.eclipse.jgit.lib.ObjectStream;
|
||||
import org.eclipse.jgit.util.IO;
|
||||
|
||||
/** Base object type accessed during revision walking. */
|
||||
public abstract class RevObject extends ObjectId {
|
||||
static final int PARSED = 1;
|
||||
|
||||
static byte[] asByteArray(RevObject obj, ObjectLoader loader) throws IOException {
|
||||
if (loader.isLarge()) {
|
||||
ObjectStream in = loader.openStream();
|
||||
try {
|
||||
long sz = in.getSize();
|
||||
if (Integer.MAX_VALUE <= sz){
|
||||
if (obj != null)
|
||||
throw new LargeObjectException(obj.copy());
|
||||
throw new LargeObjectException();
|
||||
}
|
||||
byte[] buf = new byte[(int) sz];
|
||||
IO.readFully(in, buf, 0, buf.length);
|
||||
return buf;
|
||||
} finally {
|
||||
in.close();
|
||||
}
|
||||
}
|
||||
return loader.getCachedBytes();
|
||||
}
|
||||
|
||||
int flags;
|
||||
|
||||
RevObject(final AnyObjectId name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
loadCanonical(walk);
|
||||
flags |= PARSED;
|
||||
}
|
||||
abstract void parseHeaders(RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException;
|
||||
|
||||
void parseBody(final RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if ((flags & PARSED) == 0)
|
||||
parseHeaders(walk);
|
||||
}
|
||||
|
||||
final byte[] loadCanonical(final RevWalk walk) throws IOException,
|
||||
MissingObjectException, IncorrectObjectTypeException,
|
||||
CorruptObjectException {
|
||||
return asByteArray(this, walk.reader.open(this, getType()));
|
||||
}
|
||||
abstract void parseBody(RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException;
|
||||
|
||||
/**
|
||||
* Get Git object type. See {@link Constants}.
|
||||
|
|
|
@ -131,14 +131,14 @@ protected RevTag(final AnyObjectId id) {
|
|||
@Override
|
||||
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
parseCanonical(walk, loadCanonical(walk));
|
||||
parseCanonical(walk, walk.getCachedBytes(this));
|
||||
}
|
||||
|
||||
@Override
|
||||
void parseBody(final RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if (buffer == null) {
|
||||
buffer = loadCanonical(walk);
|
||||
buffer = walk.getCachedBytes(this);
|
||||
if ((flags & PARSED) == 0)
|
||||
parseCanonical(walk, buffer);
|
||||
}
|
||||
|
|
|
@ -45,6 +45,10 @@
|
|||
|
||||
package org.eclipse.jgit.revwalk;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||
import org.eclipse.jgit.errors.MissingObjectException;
|
||||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
|
||||
|
@ -64,4 +68,20 @@ protected RevTree(final AnyObjectId id) {
|
|||
public final int getType() {
|
||||
return Constants.OBJ_TREE;
|
||||
}
|
||||
|
||||
@Override
|
||||
void parseHeaders(RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if (walk.reader.has(this))
|
||||
flags |= PARSED;
|
||||
else
|
||||
throw new MissingObjectException(this, getType());
|
||||
}
|
||||
|
||||
@Override
|
||||
void parseBody(RevWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
if ((flags & PARSED) == 0)
|
||||
parseHeaders(walk);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -61,6 +61,7 @@
|
|||
import org.eclipse.jgit.lib.AnyObjectId;
|
||||
import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
|
||||
import org.eclipse.jgit.lib.Constants;
|
||||
import org.eclipse.jgit.lib.CoreConfig;
|
||||
import org.eclipse.jgit.lib.MutableObjectId;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.ObjectIdSubclassMap;
|
||||
|
@ -170,6 +171,9 @@ public class RevWalk implements Iterable<RevCommit> {
|
|||
|
||||
private final ObjectIdSubclassMap<RevObject> objects;
|
||||
|
||||
/** Largest commit or annotated tag we are willing to touch. */
|
||||
private final int bigFileThreshold;
|
||||
|
||||
private int freeFlags = APP_FLAGS;
|
||||
|
||||
private int delayFreeFlags;
|
||||
|
@ -226,6 +230,13 @@ private RevWalk(final Repository repo, final ObjectReader or) {
|
|||
filter = RevFilter.ALL;
|
||||
treeFilter = TreeFilter.ALL;
|
||||
retainBody = true;
|
||||
|
||||
if (repo != null) {
|
||||
CoreConfig cfg = repo.getConfig().get(CoreConfig.KEY);
|
||||
bigFileThreshold = cfg.getStreamFileThreshold();
|
||||
} else {
|
||||
bigFileThreshold = 15 * 1024 * 1024;
|
||||
}
|
||||
}
|
||||
|
||||
/** @return the reader this walker is using to load objects. */
|
||||
|
@ -813,13 +824,14 @@ public RevObject parseAny(final AnyObjectId id)
|
|||
}
|
||||
|
||||
private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
||||
throws CorruptObjectException, LargeObjectException {
|
||||
throws LargeObjectException, CorruptObjectException,
|
||||
MissingObjectException, IOException {
|
||||
RevObject r;
|
||||
int type = ldr.getType();
|
||||
switch (type) {
|
||||
case Constants.OBJ_COMMIT: {
|
||||
final RevCommit c = createCommit(id);
|
||||
c.parseCanonical(this, ldr.getCachedBytes());
|
||||
c.parseCanonical(this, getCachedBytes(c, ldr));
|
||||
r = c;
|
||||
break;
|
||||
}
|
||||
|
@ -835,7 +847,7 @@ private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
|||
}
|
||||
case Constants.OBJ_TAG: {
|
||||
final RevTag t = new RevTag(id);
|
||||
t.parseCanonical(this, ldr.getCachedBytes());
|
||||
t.parseCanonical(this, getCachedBytes(t, ldr));
|
||||
r = t;
|
||||
break;
|
||||
}
|
||||
|
@ -847,6 +859,21 @@ private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
|||
return r;
|
||||
}
|
||||
|
||||
byte[] getCachedBytes(RevObject obj) throws LargeObjectException,
|
||||
MissingObjectException, IncorrectObjectTypeException, IOException {
|
||||
return getCachedBytes(obj, reader.open(obj, obj.getType()));
|
||||
}
|
||||
|
||||
byte[] getCachedBytes(RevObject obj, ObjectLoader ldr)
|
||||
throws LargeObjectException, MissingObjectException, IOException {
|
||||
try {
|
||||
return ldr.getCachedBytes(bigFileThreshold);
|
||||
} catch (LargeObjectException tooBig) {
|
||||
tooBig.setObjectId(obj);
|
||||
throw tooBig;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronous object parsing.
|
||||
*
|
||||
|
|
Loading…
Reference in New Issue