Use limited getCachedBytes in RevWalk
Parsing is rewritten to use the size limited form of getCachedBytes, thus freeing the revwalk infrastructure from needing to care about a large object vs. a small object when it gets an ObjectLoader. Right now we hardcode our upper bound for a commit or annotated tag to be 15 MiB. I don't know of any that is more than 1 MiB in the wild, so going 15x that should give us some reasonable headroom. Change-Id: If296c211d8b257d76e44908504e71dd9ba70ffa8 Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
This commit is contained in:
parent
c11711f98e
commit
127a5f95e1
|
@ -45,6 +45,10 @@
|
||||||
|
|
||||||
package org.eclipse.jgit.revwalk;
|
package org.eclipse.jgit.revwalk;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||||
|
import org.eclipse.jgit.errors.MissingObjectException;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
|
||||||
|
@ -64,4 +68,20 @@ protected RevBlob(final AnyObjectId id) {
|
||||||
public final int getType() {
|
public final int getType() {
|
||||||
return Constants.OBJ_BLOB;
|
return Constants.OBJ_BLOB;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void parseHeaders(RevWalk walk) throws MissingObjectException,
|
||||||
|
IncorrectObjectTypeException, IOException {
|
||||||
|
if (walk.reader.has(this))
|
||||||
|
flags |= PARSED;
|
||||||
|
else
|
||||||
|
throw new MissingObjectException(this, getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void parseBody(RevWalk walk) throws MissingObjectException,
|
||||||
|
IncorrectObjectTypeException, IOException {
|
||||||
|
if ((flags & PARSED) == 0)
|
||||||
|
parseHeaders(walk);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -133,14 +133,14 @@ protected RevCommit(final AnyObjectId id) {
|
||||||
@Override
|
@Override
|
||||||
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
||||||
IncorrectObjectTypeException, IOException {
|
IncorrectObjectTypeException, IOException {
|
||||||
parseCanonical(walk, loadCanonical(walk));
|
parseCanonical(walk, walk.getCachedBytes(this));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void parseBody(final RevWalk walk) throws MissingObjectException,
|
void parseBody(final RevWalk walk) throws MissingObjectException,
|
||||||
IncorrectObjectTypeException, IOException {
|
IncorrectObjectTypeException, IOException {
|
||||||
if (buffer == null) {
|
if (buffer == null) {
|
||||||
buffer = loadCanonical(walk);
|
buffer = walk.getCachedBytes(this);
|
||||||
if ((flags & PARSED) == 0)
|
if ((flags & PARSED) == 0)
|
||||||
parseCanonical(walk, buffer);
|
parseCanonical(walk, buffer);
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,64 +45,27 @@
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.eclipse.jgit.errors.CorruptObjectException;
|
|
||||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||||
import org.eclipse.jgit.errors.LargeObjectException;
|
|
||||||
import org.eclipse.jgit.errors.MissingObjectException;
|
import org.eclipse.jgit.errors.MissingObjectException;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
import org.eclipse.jgit.lib.ObjectId;
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
import org.eclipse.jgit.lib.ObjectLoader;
|
|
||||||
import org.eclipse.jgit.lib.ObjectStream;
|
|
||||||
import org.eclipse.jgit.util.IO;
|
|
||||||
|
|
||||||
/** Base object type accessed during revision walking. */
|
/** Base object type accessed during revision walking. */
|
||||||
public abstract class RevObject extends ObjectId {
|
public abstract class RevObject extends ObjectId {
|
||||||
static final int PARSED = 1;
|
static final int PARSED = 1;
|
||||||
|
|
||||||
static byte[] asByteArray(RevObject obj, ObjectLoader loader) throws IOException {
|
|
||||||
if (loader.isLarge()) {
|
|
||||||
ObjectStream in = loader.openStream();
|
|
||||||
try {
|
|
||||||
long sz = in.getSize();
|
|
||||||
if (Integer.MAX_VALUE <= sz){
|
|
||||||
if (obj != null)
|
|
||||||
throw new LargeObjectException(obj.copy());
|
|
||||||
throw new LargeObjectException();
|
|
||||||
}
|
|
||||||
byte[] buf = new byte[(int) sz];
|
|
||||||
IO.readFully(in, buf, 0, buf.length);
|
|
||||||
return buf;
|
|
||||||
} finally {
|
|
||||||
in.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return loader.getCachedBytes();
|
|
||||||
}
|
|
||||||
|
|
||||||
int flags;
|
int flags;
|
||||||
|
|
||||||
RevObject(final AnyObjectId name) {
|
RevObject(final AnyObjectId name) {
|
||||||
super(name);
|
super(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
abstract void parseHeaders(RevWalk walk) throws MissingObjectException,
|
||||||
IncorrectObjectTypeException, IOException {
|
IncorrectObjectTypeException, IOException;
|
||||||
loadCanonical(walk);
|
|
||||||
flags |= PARSED;
|
|
||||||
}
|
|
||||||
|
|
||||||
void parseBody(final RevWalk walk) throws MissingObjectException,
|
abstract void parseBody(RevWalk walk) throws MissingObjectException,
|
||||||
IncorrectObjectTypeException, IOException {
|
IncorrectObjectTypeException, IOException;
|
||||||
if ((flags & PARSED) == 0)
|
|
||||||
parseHeaders(walk);
|
|
||||||
}
|
|
||||||
|
|
||||||
final byte[] loadCanonical(final RevWalk walk) throws IOException,
|
|
||||||
MissingObjectException, IncorrectObjectTypeException,
|
|
||||||
CorruptObjectException {
|
|
||||||
return asByteArray(this, walk.reader.open(this, getType()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get Git object type. See {@link Constants}.
|
* Get Git object type. See {@link Constants}.
|
||||||
|
|
|
@ -131,14 +131,14 @@ protected RevTag(final AnyObjectId id) {
|
||||||
@Override
|
@Override
|
||||||
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
void parseHeaders(final RevWalk walk) throws MissingObjectException,
|
||||||
IncorrectObjectTypeException, IOException {
|
IncorrectObjectTypeException, IOException {
|
||||||
parseCanonical(walk, loadCanonical(walk));
|
parseCanonical(walk, walk.getCachedBytes(this));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void parseBody(final RevWalk walk) throws MissingObjectException,
|
void parseBody(final RevWalk walk) throws MissingObjectException,
|
||||||
IncorrectObjectTypeException, IOException {
|
IncorrectObjectTypeException, IOException {
|
||||||
if (buffer == null) {
|
if (buffer == null) {
|
||||||
buffer = loadCanonical(walk);
|
buffer = walk.getCachedBytes(this);
|
||||||
if ((flags & PARSED) == 0)
|
if ((flags & PARSED) == 0)
|
||||||
parseCanonical(walk, buffer);
|
parseCanonical(walk, buffer);
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,6 +45,10 @@
|
||||||
|
|
||||||
package org.eclipse.jgit.revwalk;
|
package org.eclipse.jgit.revwalk;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||||
|
import org.eclipse.jgit.errors.MissingObjectException;
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
|
||||||
|
@ -64,4 +68,20 @@ protected RevTree(final AnyObjectId id) {
|
||||||
public final int getType() {
|
public final int getType() {
|
||||||
return Constants.OBJ_TREE;
|
return Constants.OBJ_TREE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void parseHeaders(RevWalk walk) throws MissingObjectException,
|
||||||
|
IncorrectObjectTypeException, IOException {
|
||||||
|
if (walk.reader.has(this))
|
||||||
|
flags |= PARSED;
|
||||||
|
else
|
||||||
|
throw new MissingObjectException(this, getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void parseBody(RevWalk walk) throws MissingObjectException,
|
||||||
|
IncorrectObjectTypeException, IOException {
|
||||||
|
if ((flags & PARSED) == 0)
|
||||||
|
parseHeaders(walk);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -61,6 +61,7 @@
|
||||||
import org.eclipse.jgit.lib.AnyObjectId;
|
import org.eclipse.jgit.lib.AnyObjectId;
|
||||||
import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
|
import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
|
||||||
import org.eclipse.jgit.lib.Constants;
|
import org.eclipse.jgit.lib.Constants;
|
||||||
|
import org.eclipse.jgit.lib.CoreConfig;
|
||||||
import org.eclipse.jgit.lib.MutableObjectId;
|
import org.eclipse.jgit.lib.MutableObjectId;
|
||||||
import org.eclipse.jgit.lib.ObjectId;
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
import org.eclipse.jgit.lib.ObjectIdSubclassMap;
|
import org.eclipse.jgit.lib.ObjectIdSubclassMap;
|
||||||
|
@ -170,6 +171,9 @@ public class RevWalk implements Iterable<RevCommit> {
|
||||||
|
|
||||||
private final ObjectIdSubclassMap<RevObject> objects;
|
private final ObjectIdSubclassMap<RevObject> objects;
|
||||||
|
|
||||||
|
/** Largest commit or annotated tag we are willing to touch. */
|
||||||
|
private final int bigFileThreshold;
|
||||||
|
|
||||||
private int freeFlags = APP_FLAGS;
|
private int freeFlags = APP_FLAGS;
|
||||||
|
|
||||||
private int delayFreeFlags;
|
private int delayFreeFlags;
|
||||||
|
@ -226,6 +230,13 @@ private RevWalk(final Repository repo, final ObjectReader or) {
|
||||||
filter = RevFilter.ALL;
|
filter = RevFilter.ALL;
|
||||||
treeFilter = TreeFilter.ALL;
|
treeFilter = TreeFilter.ALL;
|
||||||
retainBody = true;
|
retainBody = true;
|
||||||
|
|
||||||
|
if (repo != null) {
|
||||||
|
CoreConfig cfg = repo.getConfig().get(CoreConfig.KEY);
|
||||||
|
bigFileThreshold = cfg.getStreamFileThreshold();
|
||||||
|
} else {
|
||||||
|
bigFileThreshold = 15 * 1024 * 1024;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return the reader this walker is using to load objects. */
|
/** @return the reader this walker is using to load objects. */
|
||||||
|
@ -813,13 +824,14 @@ public RevObject parseAny(final AnyObjectId id)
|
||||||
}
|
}
|
||||||
|
|
||||||
private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
||||||
throws CorruptObjectException, LargeObjectException {
|
throws LargeObjectException, CorruptObjectException,
|
||||||
|
MissingObjectException, IOException {
|
||||||
RevObject r;
|
RevObject r;
|
||||||
int type = ldr.getType();
|
int type = ldr.getType();
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case Constants.OBJ_COMMIT: {
|
case Constants.OBJ_COMMIT: {
|
||||||
final RevCommit c = createCommit(id);
|
final RevCommit c = createCommit(id);
|
||||||
c.parseCanonical(this, ldr.getCachedBytes());
|
c.parseCanonical(this, getCachedBytes(c, ldr));
|
||||||
r = c;
|
r = c;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -835,7 +847,7 @@ private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
||||||
}
|
}
|
||||||
case Constants.OBJ_TAG: {
|
case Constants.OBJ_TAG: {
|
||||||
final RevTag t = new RevTag(id);
|
final RevTag t = new RevTag(id);
|
||||||
t.parseCanonical(this, ldr.getCachedBytes());
|
t.parseCanonical(this, getCachedBytes(t, ldr));
|
||||||
r = t;
|
r = t;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -847,6 +859,21 @@ private RevObject parseNew(AnyObjectId id, ObjectLoader ldr)
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
byte[] getCachedBytes(RevObject obj) throws LargeObjectException,
|
||||||
|
MissingObjectException, IncorrectObjectTypeException, IOException {
|
||||||
|
return getCachedBytes(obj, reader.open(obj, obj.getType()));
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[] getCachedBytes(RevObject obj, ObjectLoader ldr)
|
||||||
|
throws LargeObjectException, MissingObjectException, IOException {
|
||||||
|
try {
|
||||||
|
return ldr.getCachedBytes(bigFileThreshold);
|
||||||
|
} catch (LargeObjectException tooBig) {
|
||||||
|
tooBig.setObjectId(obj);
|
||||||
|
throw tooBig;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Asynchronous object parsing.
|
* Asynchronous object parsing.
|
||||||
*
|
*
|
||||||
|
|
Loading…
Reference in New Issue