Tighten up local packed object representation during packing
Rather than making a loader, and then using that to fill the object representation, parse the header and set up our data directly. This saves some time, as we don't waste cycles on information we won't use right now. The weight computed for a representation is now its actual stored size in the pack file, rather than its inflated size. This accounts for changes made when the compression level is modified on the repository. It is however more costly to determine the weight of the object, since we have to find its length in the pack. To try and recover that cost we now cache the length as part of our ObjectToPack record, so it doesn't have to be found during the output phase. A LocalObjectToPack now costs us (assuming 32 bit pointers): (32 bit) (64 bit) vm header: 8 bytes 8 bytes ObjectId: 20 bytes 20 bytes PackedObjectInfo: 12 bytes 12 bytes ObjectToPack: 8 bytes 12 bytes LocalOTP: 20 bytes 24 bytes ----------- --------- 68 bytes 74 bytes Change-Id: I923d2736186eb2ac8ab498d3eb137e17930fcb50 Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
This commit is contained in:
parent
ad5238dc67
commit
86547022f0
|
@ -49,35 +49,70 @@
|
|||
import org.eclipse.jgit.lib.StoredObjectRepresentation;
|
||||
|
||||
class LocalObjectRepresentation extends StoredObjectRepresentation {
|
||||
final PackedObjectLoader ldr;
|
||||
|
||||
LocalObjectRepresentation(PackedObjectLoader ldr) {
|
||||
this.ldr = ldr;
|
||||
static LocalObjectRepresentation newWhole(PackFile f, long p, long length) {
|
||||
LocalObjectRepresentation r = new LocalObjectRepresentation() {
|
||||
@Override
|
||||
public int getFormat() {
|
||||
return PACK_WHOLE;
|
||||
}
|
||||
};
|
||||
r.pack = f;
|
||||
r.offset = p;
|
||||
r.length = length;
|
||||
return r;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getFormat() {
|
||||
if (ldr instanceof DeltaPackedObjectLoader)
|
||||
return PACK_DELTA;
|
||||
if (ldr instanceof WholePackedObjectLoader)
|
||||
return PACK_WHOLE;
|
||||
return FORMAT_OTHER;
|
||||
static LocalObjectRepresentation newDelta(PackFile f, long p, long n,
|
||||
ObjectId base) {
|
||||
LocalObjectRepresentation r = new Delta();
|
||||
r.pack = f;
|
||||
r.offset = p;
|
||||
r.length = n;
|
||||
r.baseId = base;
|
||||
return r;
|
||||
}
|
||||
|
||||
static LocalObjectRepresentation newDelta(PackFile f, long p, long n,
|
||||
long base) {
|
||||
LocalObjectRepresentation r = new Delta();
|
||||
r.pack = f;
|
||||
r.offset = p;
|
||||
r.length = n;
|
||||
r.baseOffset = base;
|
||||
return r;
|
||||
}
|
||||
|
||||
PackFile pack;
|
||||
|
||||
long offset;
|
||||
|
||||
long length;
|
||||
|
||||
private long baseOffset;
|
||||
|
||||
private ObjectId baseId;
|
||||
|
||||
@Override
|
||||
public int getWeight() {
|
||||
long sz = ldr.getRawSize();
|
||||
if (Integer.MAX_VALUE < sz)
|
||||
return WEIGHT_UNKNOWN;
|
||||
return (int) sz;
|
||||
return (int) Math.min(length, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ObjectId getDeltaBase() {
|
||||
try {
|
||||
return ldr.getDeltaBase();
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
if (baseId == null && getFormat() == PACK_DELTA) {
|
||||
try {
|
||||
baseId = pack.findObjectForOffset(baseOffset);
|
||||
} catch (IOException error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return baseId;
|
||||
}
|
||||
|
||||
private static final class Delta extends LocalObjectRepresentation {
|
||||
@Override
|
||||
public int getFormat() {
|
||||
return PACK_DELTA;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -50,10 +50,13 @@
|
|||
/** {@link ObjectToPack} for {@link ObjectDirectory}. */
|
||||
class LocalObjectToPack extends ObjectToPack {
|
||||
/** Pack to reuse compressed data from, otherwise null. */
|
||||
PackFile copyFromPack;
|
||||
PackFile pack;
|
||||
|
||||
/** Offset of the object's header in {@link #copyFromPack}. */
|
||||
long copyOffset;
|
||||
/** Offset of the object's header in {@link #pack}. */
|
||||
long offset;
|
||||
|
||||
/** Length of the data section of the object. */
|
||||
long length;
|
||||
|
||||
LocalObjectToPack(RevObject obj) {
|
||||
super(obj);
|
||||
|
@ -61,8 +64,9 @@ class LocalObjectToPack extends ObjectToPack {
|
|||
|
||||
@Override
|
||||
public void select(StoredObjectRepresentation ref) {
|
||||
LocalObjectRepresentation ptr = (LocalObjectRepresentation)ref;
|
||||
this.copyFromPack = ptr.ldr.pack;
|
||||
this.copyOffset = ptr.ldr.objectOffset;
|
||||
LocalObjectRepresentation ptr = (LocalObjectRepresentation) ref;
|
||||
this.pack = ptr.pack;
|
||||
this.offset = ptr.offset;
|
||||
this.length = ptr.length;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -299,9 +299,9 @@ void selectObjectRepresentation(PackWriter packer, ObjectToPack otp,
|
|||
SEARCH: for (;;) {
|
||||
for (final PackFile p : pList.packs) {
|
||||
try {
|
||||
PackedObjectLoader ldr = p.get(curs, otp);
|
||||
if (ldr != null)
|
||||
packer.select(otp, new LocalObjectRepresentation(ldr));
|
||||
LocalObjectRepresentation rep = p.representation(curs, otp);
|
||||
if (rep != null)
|
||||
packer.select(otp, rep);
|
||||
} catch (PackMismatchException e) {
|
||||
// Pack was modified; refresh the entire pack list.
|
||||
//
|
||||
|
|
|
@ -309,7 +309,7 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
|
||||
// Rip apart the header so we can discover the size.
|
||||
//
|
||||
readFully(src.copyOffset, buf, 0, 20, curs);
|
||||
readFully(src.offset, buf, 0, 20, curs);
|
||||
int c = buf[0] & 0xff;
|
||||
final int typeCode = (c >> 4) & 7;
|
||||
long inflatedLength = c & 15;
|
||||
|
@ -331,7 +331,7 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
crc1.update(buf, 0, headerCnt);
|
||||
crc2.update(buf, 0, headerCnt);
|
||||
|
||||
readFully(src.copyOffset + headerCnt, buf, 0, 20, curs);
|
||||
readFully(src.offset + headerCnt, buf, 0, 20, curs);
|
||||
crc1.update(buf, 0, 20);
|
||||
crc2.update(buf, 0, headerCnt);
|
||||
headerCnt += 20;
|
||||
|
@ -340,8 +340,8 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
crc2.update(buf, 0, headerCnt);
|
||||
}
|
||||
|
||||
final long dataOffset = src.copyOffset + headerCnt;
|
||||
final long dataLength;
|
||||
final long dataOffset = src.offset + headerCnt;
|
||||
final long dataLength = src.length;
|
||||
final long expectedCRC;
|
||||
final ByteArrayWindow quickCopy;
|
||||
|
||||
|
@ -349,7 +349,6 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
// we report it missing instead.
|
||||
//
|
||||
try {
|
||||
dataLength = findEndOffset(src.copyOffset) - dataOffset;
|
||||
quickCopy = curs.quickCopy(this, dataOffset, dataLength);
|
||||
|
||||
if (idx().hasCRC32Support()) {
|
||||
|
@ -370,10 +369,10 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
}
|
||||
}
|
||||
if (crc1.getValue() != expectedCRC) {
|
||||
setCorrupt(src.copyOffset);
|
||||
setCorrupt(src.offset);
|
||||
throw new CorruptObjectException(MessageFormat.format(
|
||||
JGitText.get().objectAtHasBadZlibStream,
|
||||
src.copyOffset, getPackFile()));
|
||||
src.offset, getPackFile()));
|
||||
}
|
||||
} else {
|
||||
// We don't have a CRC32 code in the index, so compute it
|
||||
|
@ -399,20 +398,20 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
}
|
||||
}
|
||||
if (!inf.finished() || inf.getBytesRead() != dataLength) {
|
||||
setCorrupt(src.copyOffset);
|
||||
setCorrupt(src.offset);
|
||||
throw new EOFException(MessageFormat.format(
|
||||
JGitText.get().shortCompressedStreamAt,
|
||||
src.copyOffset));
|
||||
src.offset));
|
||||
}
|
||||
expectedCRC = crc1.getValue();
|
||||
}
|
||||
} catch (DataFormatException dataFormat) {
|
||||
setCorrupt(src.copyOffset);
|
||||
setCorrupt(src.offset);
|
||||
|
||||
CorruptObjectException corruptObject = new CorruptObjectException(
|
||||
MessageFormat.format(
|
||||
JGitText.get().objectAtHasBadZlibStream,
|
||||
src.copyOffset, getPackFile()));
|
||||
src.offset, getPackFile()));
|
||||
corruptObject.initCause(dataFormat);
|
||||
|
||||
StoredObjectRepresentationNotAvailableException gone;
|
||||
|
@ -458,7 +457,7 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
|||
}
|
||||
if (crc2.getValue() != expectedCRC) {
|
||||
throw new CorruptObjectException(MessageFormat.format(JGitText
|
||||
.get().objectAtHasBadZlibStream, src.copyOffset,
|
||||
.get().objectAtHasBadZlibStream, src.offset,
|
||||
getPackFile()));
|
||||
}
|
||||
}
|
||||
|
@ -661,6 +660,55 @@ private PackedObjectLoader reader(final WindowCursor curs,
|
|||
}
|
||||
}
|
||||
|
||||
LocalObjectRepresentation representation(final WindowCursor curs,
|
||||
final AnyObjectId objectId) throws IOException {
|
||||
final long pos = idx().findOffset(objectId);
|
||||
if (pos < 0)
|
||||
return null;
|
||||
|
||||
final byte[] ib = curs.tempId;
|
||||
readFully(pos, ib, 0, 20, curs);
|
||||
int c = ib[0] & 0xff;
|
||||
int p = 1;
|
||||
final int typeCode = (c >> 4) & 7;
|
||||
while ((c & 0x80) != 0)
|
||||
c = ib[p++] & 0xff;
|
||||
|
||||
long len = (findEndOffset(pos) - pos);
|
||||
switch (typeCode) {
|
||||
case Constants.OBJ_COMMIT:
|
||||
case Constants.OBJ_TREE:
|
||||
case Constants.OBJ_BLOB:
|
||||
case Constants.OBJ_TAG:
|
||||
return LocalObjectRepresentation.newWhole(this, pos, len - p);
|
||||
|
||||
case Constants.OBJ_OFS_DELTA: {
|
||||
c = ib[p++] & 0xff;
|
||||
long ofs = c & 127;
|
||||
while ((c & 128) != 0) {
|
||||
ofs += 1;
|
||||
c = ib[p++] & 0xff;
|
||||
ofs <<= 7;
|
||||
ofs += (c & 127);
|
||||
}
|
||||
ofs = pos - ofs;
|
||||
return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs);
|
||||
}
|
||||
|
||||
case Constants.OBJ_REF_DELTA: {
|
||||
len -= p;
|
||||
len -= Constants.OBJECT_ID_LENGTH;
|
||||
readFully(pos + p, ib, 0, 20, curs);
|
||||
ObjectId id = ObjectId.fromRaw(ib);
|
||||
return LocalObjectRepresentation.newDelta(this, pos, len, id);
|
||||
}
|
||||
|
||||
default:
|
||||
throw new IOException(MessageFormat.format(
|
||||
JGitText.get().unknownObjectType, typeCode));
|
||||
}
|
||||
}
|
||||
|
||||
private long findEndOffset(final long startOffset)
|
||||
throws IOException, CorruptObjectException {
|
||||
final long maxOffset = length - 20;
|
||||
|
|
|
@ -103,7 +103,7 @@ public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp)
|
|||
public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp)
|
||||
throws IOException, StoredObjectRepresentationNotAvailableException {
|
||||
LocalObjectToPack src = (LocalObjectToPack) otp;
|
||||
src.copyFromPack.copyAsIs(out, src, this);
|
||||
src.pack.copyAsIs(out, src, this);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
Loading…
Reference in New Issue