From b584cb8754796bb96526b37b26b517741b9c9d1c Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 8 Jul 2010 17:08:55 -0700 Subject: [PATCH] Add getObjectSize to ObjectReader This is an informational function used by PackWriter to help it better organize objects for delta compression. Storage systems can implement it to provide up more detailed size information, or they can simply rely on the default behavior that uses the ObjectLoader obtained from open. For local file storage, we can obtain this information faster through specialized routines that parse a pack object header. Change-Id: I13a09b4effb71ea5151b51547f7d091564531e58 Signed-off-by: Shawn O. Pearce --- .../org/eclipse/jgit/lib/ObjectReader.java | 28 ++++++++++ .../storage/file/CachedObjectDirectory.java | 14 +++++ .../jgit/storage/file/FileObjectDatabase.java | 55 +++++++++++++++++++ .../jgit/storage/file/ObjectDirectory.java | 40 ++++++++++++++ .../eclipse/jgit/storage/file/PackFile.java | 53 ++++++++++++++++++ .../jgit/storage/file/UnpackedObject.java | 46 +++++++++++++++- .../jgit/storage/file/WindowCursor.java | 12 ++++ 7 files changed, 247 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java index 1af3cb2de..d0351f867 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java @@ -135,6 +135,34 @@ public abstract ObjectLoader open(AnyObjectId objectId, int typeHint) throws MissingObjectException, IncorrectObjectTypeException, IOException; + /** + * Get only the size of an object. + *

+ * The default implementation of this method opens an ObjectLoader. + * Databases are encouraged to override this if a faster access method is + * available to them. + * + * @param objectId + * identity of the object to open. + * @param typeHint + * hint about the type of object being requested; + * {@link #OBJ_ANY} if the object type is not known, or does not + * matter to the caller. + * @return size of object in bytes. + * @throws MissingObjectException + * the object does not exist. + * @throws IncorrectObjectTypeException + * typeHint was not OBJ_ANY, and the object's actual type does + * not match typeHint. + * @throws IOException + * the object store cannot be accessed. + */ + public long getObjectSize(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + return open(objectId, typeHint).getSize(); + } + /** * Release any resources used by this reader. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java index 505850c42..8ea0b854c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/CachedObjectDirectory.java @@ -186,6 +186,20 @@ ObjectLoader openObject2(WindowCursor curs, String objectName, throw new UnsupportedOperationException(); } + @Override + long getObjectSize1(WindowCursor curs, AnyObjectId objectId) throws IOException { + if (unpackedObjects.contains(objectId)) + return wrapped.getObjectSize2(curs, objectId.name(), objectId); + return wrapped.getObjectSize1(curs, objectId); + } + + @Override + long getObjectSize2(WindowCursor curs, String objectName, AnyObjectId objectId) + throws IOException { + // This method should never be invoked. + throw new UnsupportedOperationException(); + } + @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java index 444fd809b..250c7cac0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileObjectDatabase.java @@ -166,6 +166,55 @@ final ObjectLoader openObjectImpl2(final WindowCursor curs, return null; } + long getObjectSize(WindowCursor curs, AnyObjectId objectId) + throws IOException { + long sz = getObjectSizeImpl1(curs, objectId); + if (0 <= sz) + return sz; + return getObjectSizeImpl2(curs, objectId.name(), objectId); + } + + final long getObjectSizeImpl1(final WindowCursor curs, + final AnyObjectId objectId) throws IOException { + long sz; + + sz = getObjectSize1(curs, objectId); + if (0 <= sz) + return sz; + + for (final AlternateHandle alt : myAlternates()) { + sz = alt.db.getObjectSizeImpl1(curs, objectId); + if (0 <= sz) + return sz; + } + + if (tryAgain1()) { + sz = getObjectSize1(curs, objectId); + if (0 <= sz) + return sz; + } + + return -1; + } + + final long getObjectSizeImpl2(final WindowCursor curs, + final String objectName, final AnyObjectId objectId) + throws IOException { + long sz; + + sz = getObjectSize2(curs, objectName, objectId); + if (0 <= sz) + return sz; + + for (final AlternateHandle alt : myAlternates()) { + sz = alt.db.getObjectSizeImpl2(curs, objectName, objectId); + if (0 <= sz) + return sz; + } + + return -1; + } + abstract void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException; @@ -185,6 +234,12 @@ abstract ObjectLoader openObject1(WindowCursor curs, AnyObjectId objectId) abstract ObjectLoader openObject2(WindowCursor curs, String objectName, AnyObjectId objectId) throws IOException; + abstract long getObjectSize1(WindowCursor curs, AnyObjectId objectId) + throws IOException; + + abstract long getObjectSize2(WindowCursor curs, String objectName, + AnyObjectId objectId) throws IOException; + abstract FileObjectDatabase newCachedFileObjectDatabase(); abstract int getStreamFileThreshold(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java index 8177155f4..6fe4fd754 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java @@ -304,6 +304,46 @@ ObjectLoader openObject1(final WindowCursor curs, } } + long getObjectSize1(final WindowCursor curs, final AnyObjectId objectId) + throws IOException { + PackList pList = packList.get(); + SEARCH: for (;;) { + for (final PackFile p : pList.packs) { + try { + long sz = p.getObjectSize(curs, objectId); + if (0 <= sz) + return sz; + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + // + pList = scanPacks(pList); + continue SEARCH; + } catch (IOException e) { + // Assume the pack is corrupted. + // + removePack(p); + } + } + return -1; + } + } + + @Override + long getObjectSize2(WindowCursor curs, String objectName, + AnyObjectId objectId) throws IOException { + try { + File path = fileFor(objectName); + FileInputStream in = new FileInputStream(path); + try { + return UnpackedObject.getSize(in, objectId, curs); + } finally { + in.close(); + } + } catch (FileNotFoundException noFile) { + return -1; + } + } + @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java index 40bb07100..e74a7c014 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/PackFile.java @@ -779,6 +779,59 @@ int getObjectType(final WindowCursor curs, long pos) throws IOException { } } + long getObjectSize(final WindowCursor curs, final AnyObjectId id) + throws IOException { + final long offset = idx().findOffset(id); + return 0 < offset ? getObjectSize(curs, offset) : -1; + } + + long getObjectSize(final WindowCursor curs, final long pos) + throws IOException { + final byte[] ib = curs.tempId; + readFully(pos, ib, 0, 20, curs); + int c = ib[0] & 0xff; + final int type = (c >> 4) & 7; + long sz = c & 15; + int shift = 4; + int p = 1; + while ((c & 0x80) != 0) { + c = ib[p++] & 0xff; + sz += (c & 0x7f) << shift; + shift += 7; + } + + long deltaAt; + switch (type) { + case Constants.OBJ_COMMIT: + case Constants.OBJ_TREE: + case Constants.OBJ_BLOB: + case Constants.OBJ_TAG: + return sz; + + case Constants.OBJ_OFS_DELTA: + c = ib[p++] & 0xff; + while ((c & 128) != 0) + c = ib[p++] & 0xff; + deltaAt = pos + p; + break; + + case Constants.OBJ_REF_DELTA: + deltaAt = pos + p + 20; + break; + + default: + throw new IOException(MessageFormat.format( + JGitText.get().unknownObjectType, type)); + } + + try { + return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt)); + } catch (DataFormatException e) { + throw new CorruptObjectException(MessageFormat.format(JGitText + .get().objectAtHasBadZlibStream, pos, getPackFile())); + } + } + LocalObjectRepresentation representation(final WindowCursor curs, final AnyObjectId objectId) throws IOException { final long pos = idx().findOffset(objectId); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java index 0325c5711..59f9c8267 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/UnpackedObject.java @@ -109,7 +109,7 @@ static ObjectLoader open(InputStream in, File path, AnyObjectId id, if (isStandardFormat(hdr)) { in.reset(); - Inflater inf = wc.inflater(); + Inflater inf = wc.inflater(); InputStream zIn = inflate(in, inf); int avail = readSome(zIn, hdr, 0, 64); if (avail < 5) @@ -183,6 +183,50 @@ static ObjectLoader open(InputStream in, File path, AnyObjectId id, } } + static long getSize(InputStream in, AnyObjectId id, WindowCursor wc) + throws IOException { + try { + in = buffer(in); + in.mark(20); + final byte[] hdr = new byte[64]; + IO.readFully(in, hdr, 0, 2); + + if (isStandardFormat(hdr)) { + in.reset(); + Inflater inf = wc.inflater(); + InputStream zIn = inflate(in, inf); + int avail = readSome(zIn, hdr, 0, 64); + if (avail < 5) + throw new CorruptObjectException(id, + JGitText.get().corruptObjectNoHeader); + + final MutableInteger p = new MutableInteger(); + Constants.decodeTypeString(id, hdr, (byte) ' ', p); + long size = RawParseUtils.parseLongBase10(hdr, p.value, p); + if (size < 0) + throw new CorruptObjectException(id, + JGitText.get().corruptObjectNegativeSize); + return size; + + } else { + readSome(in, hdr, 2, 18); + int c = hdr[0] & 0xff; + long size = c & 15; + int shift = 4; + int p = 1; + while ((c & 0x80) != 0) { + c = hdr[p++] & 0xff; + size += (c & 0x7f) << shift; + shift += 7; + } + return size; + } + } catch (ZipException badStream) { + throw new CorruptObjectException(id, + JGitText.get().corruptObjectBadStream); + } + } + private static void checkValidEndOfStream(InputStream in, Inflater inf, AnyObjectId id, final byte[] buf) throws IOException, CorruptObjectException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java index 6f4e72a82..04ee8b2c4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java @@ -95,6 +95,18 @@ public ObjectLoader open(AnyObjectId objectId, int typeHint) return ldr; } + public long getObjectSize(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + long sz = db.getObjectSize(this, objectId); + if (sz < 0) { + if (typeHint == OBJ_ANY) + throw new MissingObjectException(objectId.copy(), "unknown"); + throw new MissingObjectException(objectId.copy(), typeHint); + } + return sz; + } + public LocalObjectToPack newObjectToPack(RevObject obj) { return new LocalObjectToPack(obj); }