Merge "ReceivePack supports InputStream data after pack"
This commit is contained in:
commit
8712926958
|
@ -304,6 +304,154 @@ public void testMaxObjectSizeDeltaResultSize() throws Exception {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNonMarkingInputStream() throws Exception {
|
||||||
|
TestRepository d = new TestRepository(db);
|
||||||
|
RevBlob a = d.blob("a");
|
||||||
|
|
||||||
|
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
|
||||||
|
packHeader(pack, 1);
|
||||||
|
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
|
||||||
|
a.copyRawTo(pack);
|
||||||
|
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
|
||||||
|
digest(pack);
|
||||||
|
|
||||||
|
InputStream in = new ByteArrayInputStream(pack.toByteArray()) {
|
||||||
|
@Override
|
||||||
|
public boolean markSupported() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void mark(int maxlength) {
|
||||||
|
fail("Mark should not be called");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
PackParser p = index(in);
|
||||||
|
p.setAllowThin(true);
|
||||||
|
p.setCheckEofAfterPackFooter(false);
|
||||||
|
p.setExpectDataAfterPackFooter(true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
p.parse(NullProgressMonitor.INSTANCE);
|
||||||
|
fail("PackParser should have failed");
|
||||||
|
} catch (IOException e) {
|
||||||
|
assertEquals(e.getMessage(),
|
||||||
|
JGitText.get().inputStreamMustSupportMark);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDataAfterPackFooterSingleRead() throws Exception {
|
||||||
|
TestRepository d = new TestRepository(db);
|
||||||
|
RevBlob a = d.blob("a");
|
||||||
|
|
||||||
|
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32*1024);
|
||||||
|
packHeader(pack, 1);
|
||||||
|
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
|
||||||
|
a.copyRawTo(pack);
|
||||||
|
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
|
||||||
|
digest(pack);
|
||||||
|
|
||||||
|
byte packData[] = pack.toByteArray();
|
||||||
|
byte streamData[] = new byte[packData.length + 1];
|
||||||
|
System.arraycopy(packData, 0, streamData, 0, packData.length);
|
||||||
|
streamData[packData.length] = 0x7e;
|
||||||
|
|
||||||
|
InputStream in = new ByteArrayInputStream(streamData);
|
||||||
|
PackParser p = index(in);
|
||||||
|
p.setAllowThin(true);
|
||||||
|
p.setCheckEofAfterPackFooter(false);
|
||||||
|
p.setExpectDataAfterPackFooter(true);
|
||||||
|
|
||||||
|
p.parse(NullProgressMonitor.INSTANCE);
|
||||||
|
|
||||||
|
assertEquals(0x7e, in.read());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDataAfterPackFooterSplitObjectRead() throws Exception {
|
||||||
|
final byte[] data = Constants.encode("0123456789");
|
||||||
|
|
||||||
|
// Build a pack ~17k
|
||||||
|
int objects = 900;
|
||||||
|
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
|
||||||
|
packHeader(pack, objects);
|
||||||
|
|
||||||
|
for (int i = 0; i < objects; i++) {
|
||||||
|
pack.write((Constants.OBJ_BLOB) << 4 | 10);
|
||||||
|
deflate(pack, data);
|
||||||
|
}
|
||||||
|
digest(pack);
|
||||||
|
|
||||||
|
byte packData[] = pack.toByteArray();
|
||||||
|
byte streamData[] = new byte[packData.length + 1];
|
||||||
|
System.arraycopy(packData, 0, streamData, 0, packData.length);
|
||||||
|
streamData[packData.length] = 0x7e;
|
||||||
|
InputStream in = new ByteArrayInputStream(streamData);
|
||||||
|
PackParser p = index(in);
|
||||||
|
p.setAllowThin(true);
|
||||||
|
p.setCheckEofAfterPackFooter(false);
|
||||||
|
p.setExpectDataAfterPackFooter(true);
|
||||||
|
|
||||||
|
p.parse(NullProgressMonitor.INSTANCE);
|
||||||
|
|
||||||
|
assertEquals(0x7e, in.read());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
|
||||||
|
TestRepository d = new TestRepository(db);
|
||||||
|
final byte[] data = Constants.encode("a");
|
||||||
|
RevBlob b = d.blob(data);
|
||||||
|
|
||||||
|
int objects = 248;
|
||||||
|
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
|
||||||
|
packHeader(pack, objects + 1);
|
||||||
|
int offset = 13;
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (int i = 0; i < offset; i++)
|
||||||
|
sb.append(i);
|
||||||
|
offset = sb.toString().length();
|
||||||
|
int lenByte = (Constants.OBJ_BLOB) << 4 | (offset & 0x0F);
|
||||||
|
offset >>= 4;
|
||||||
|
if (offset > 0)
|
||||||
|
lenByte |= 1 << 7;
|
||||||
|
pack.write(lenByte);
|
||||||
|
while (offset > 0) {
|
||||||
|
lenByte = offset & 0x7F;
|
||||||
|
offset >>= 6;
|
||||||
|
if (offset > 0)
|
||||||
|
lenByte |= 1 << 7;
|
||||||
|
pack.write(lenByte);
|
||||||
|
}
|
||||||
|
deflate(pack, Constants.encode(sb.toString()));
|
||||||
|
|
||||||
|
for (int i = 0; i < objects; i++) {
|
||||||
|
// The last pack header written falls across the 8192 byte boundary
|
||||||
|
// between [8189:8210]
|
||||||
|
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
|
||||||
|
b.copyRawTo(pack);
|
||||||
|
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
|
||||||
|
}
|
||||||
|
digest(pack);
|
||||||
|
|
||||||
|
byte packData[] = pack.toByteArray();
|
||||||
|
byte streamData[] = new byte[packData.length + 1];
|
||||||
|
System.arraycopy(packData, 0, streamData, 0, packData.length);
|
||||||
|
streamData[packData.length] = 0x7e;
|
||||||
|
InputStream in = new ByteArrayInputStream(streamData);
|
||||||
|
PackParser p = index(in);
|
||||||
|
p.setAllowThin(true);
|
||||||
|
p.setCheckEofAfterPackFooter(false);
|
||||||
|
p.setExpectDataAfterPackFooter(true);
|
||||||
|
|
||||||
|
p.parse(NullProgressMonitor.INSTANCE);
|
||||||
|
|
||||||
|
assertEquals(0x7e, in.read());
|
||||||
|
}
|
||||||
|
|
||||||
private void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
|
private void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
final byte[] hdr = new byte[8];
|
final byte[] hdr = new byte[8];
|
||||||
|
|
|
@ -224,6 +224,7 @@ indexFileIsTooLargeForJgit=Index file is too large for jgit
|
||||||
indexSignatureIsInvalid=Index signature is invalid: {0}
|
indexSignatureIsInvalid=Index signature is invalid: {0}
|
||||||
indexWriteException=Modified index could not be written
|
indexWriteException=Modified index could not be written
|
||||||
inMemoryBufferLimitExceeded=In-memory buffer limit exceeded
|
inMemoryBufferLimitExceeded=In-memory buffer limit exceeded
|
||||||
|
inputStreamMustSupportMark=InputStream must support mark()
|
||||||
integerValueOutOfRange=Integer value {0}.{1} out of range
|
integerValueOutOfRange=Integer value {0}.{1} out of range
|
||||||
internalRevisionError=internal revision error
|
internalRevisionError=internal revision error
|
||||||
internalServerError=internal server error
|
internalServerError=internal server error
|
||||||
|
|
|
@ -284,6 +284,7 @@ public static JGitText get() {
|
||||||
/***/ public String indexSignatureIsInvalid;
|
/***/ public String indexSignatureIsInvalid;
|
||||||
/***/ public String indexWriteException;
|
/***/ public String indexWriteException;
|
||||||
/***/ public String inMemoryBufferLimitExceeded;
|
/***/ public String inMemoryBufferLimitExceeded;
|
||||||
|
/***/ public String inputStreamMustSupportMark;
|
||||||
/***/ public String integerValueOutOfRange;
|
/***/ public String integerValueOutOfRange;
|
||||||
/***/ public String internalRevisionError;
|
/***/ public String internalRevisionError;
|
||||||
/***/ public String internalServerError;
|
/***/ public String internalServerError;
|
||||||
|
|
|
@ -154,6 +154,9 @@ public Set<String> getCapabilities() {
|
||||||
*/
|
*/
|
||||||
protected boolean biDirectionalPipe = true;
|
protected boolean biDirectionalPipe = true;
|
||||||
|
|
||||||
|
/** Expecting data after the pack footer */
|
||||||
|
protected boolean expectDataAfterPackFooter;
|
||||||
|
|
||||||
/** Should an incoming transfer validate objects? */
|
/** Should an incoming transfer validate objects? */
|
||||||
protected boolean checkReceivedObjects;
|
protected boolean checkReceivedObjects;
|
||||||
|
|
||||||
|
@ -454,6 +457,19 @@ public void setBiDirectionalPipe(final boolean twoWay) {
|
||||||
biDirectionalPipe = twoWay;
|
biDirectionalPipe = twoWay;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @return true if there is data expected after the pack footer. */
|
||||||
|
public boolean isExpectDataAfterPackFooter() {
|
||||||
|
return expectDataAfterPackFooter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param e
|
||||||
|
* true if there is additional data in InputStream after pack.
|
||||||
|
*/
|
||||||
|
public void setExpectDataAfterPackFooter(boolean e) {
|
||||||
|
expectDataAfterPackFooter = e;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return true if this instance will verify received objects are formatted
|
* @return true if this instance will verify received objects are formatted
|
||||||
* correctly. Validating objects requires more CPU time on this side
|
* correctly. Validating objects requires more CPU time on this side
|
||||||
|
@ -909,6 +925,7 @@ private void receivePack() throws IOException {
|
||||||
parser.setNeedNewObjectIds(checkReferencedIsReachable);
|
parser.setNeedNewObjectIds(checkReferencedIsReachable);
|
||||||
parser.setNeedBaseObjectIds(checkReferencedIsReachable);
|
parser.setNeedBaseObjectIds(checkReferencedIsReachable);
|
||||||
parser.setCheckEofAfterPackFooter(!biDirectionalPipe);
|
parser.setCheckEofAfterPackFooter(!biDirectionalPipe);
|
||||||
|
parser.setExpectDataAfterPackFooter(isExpectDataAfterPackFooter());
|
||||||
parser.setObjectChecking(isCheckReceivedObjects());
|
parser.setObjectChecking(isCheckReceivedObjects());
|
||||||
parser.setLockMessage(lockMsg);
|
parser.setLockMessage(lockMsg);
|
||||||
parser.setMaxObjectSizeLimit(maxObjectSizeLimit);
|
parser.setMaxObjectSizeLimit(maxObjectSizeLimit);
|
||||||
|
|
|
@ -141,6 +141,8 @@ public static enum Source {
|
||||||
|
|
||||||
private boolean checkEofAfterPackFooter;
|
private boolean checkEofAfterPackFooter;
|
||||||
|
|
||||||
|
private boolean expectDataAfterPackFooter;
|
||||||
|
|
||||||
private long objectCount;
|
private long objectCount;
|
||||||
|
|
||||||
private PackedObjectInfo[] entries;
|
private PackedObjectInfo[] entries;
|
||||||
|
@ -305,6 +307,21 @@ public void setCheckEofAfterPackFooter(boolean b) {
|
||||||
checkEofAfterPackFooter = b;
|
checkEofAfterPackFooter = b;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** @return true if there is data expected after the pack footer. */
|
||||||
|
public boolean isExpectDataAfterPackFooter() {
|
||||||
|
return expectDataAfterPackFooter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param e
|
||||||
|
* true if there is additional data in InputStream after pack.
|
||||||
|
* This requires the InputStream to support the mark and reset
|
||||||
|
* functions.
|
||||||
|
*/
|
||||||
|
public void setExpectDataAfterPackFooter(boolean e) {
|
||||||
|
expectDataAfterPackFooter = e;
|
||||||
|
}
|
||||||
|
|
||||||
/** @return the new objects that were sent by the user */
|
/** @return the new objects that were sent by the user */
|
||||||
public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
|
public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
|
||||||
if (newObjectIds != null)
|
if (newObjectIds != null)
|
||||||
|
@ -826,6 +843,13 @@ private void growEntries(int extraObjects) {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void readPackHeader() throws IOException {
|
private void readPackHeader() throws IOException {
|
||||||
|
if (expectDataAfterPackFooter) {
|
||||||
|
if (!in.markSupported())
|
||||||
|
throw new IOException(
|
||||||
|
JGitText.get().inputStreamMustSupportMark);
|
||||||
|
in.mark(buf.length);
|
||||||
|
}
|
||||||
|
|
||||||
final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
|
final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
|
||||||
final int p = fill(Source.INPUT, hdrln);
|
final int p = fill(Source.INPUT, hdrln);
|
||||||
for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
|
for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
|
||||||
|
@ -851,23 +875,19 @@ private void readPackFooter() throws IOException {
|
||||||
System.arraycopy(buf, c, srcHash, 0, 20);
|
System.arraycopy(buf, c, srcHash, 0, 20);
|
||||||
use(20);
|
use(20);
|
||||||
|
|
||||||
// The input stream should be at EOF at this point. We do not support
|
if (bAvail != 0 && !expectDataAfterPackFooter)
|
||||||
// yielding back any remaining buffered data after the pack footer, so
|
|
||||||
// protocols that embed a pack stream are required to either end their
|
|
||||||
// stream with the pack, or embed the pack with a framing system like
|
|
||||||
// the SideBandInputStream does.
|
|
||||||
|
|
||||||
if (bAvail != 0)
|
|
||||||
throw new CorruptObjectException(MessageFormat.format(
|
throw new CorruptObjectException(MessageFormat.format(
|
||||||
JGitText.get().expectedEOFReceived,
|
JGitText.get().expectedEOFReceived,
|
||||||
"\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
|
"\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
|
||||||
|
|
||||||
if (isCheckEofAfterPackFooter()) {
|
if (isCheckEofAfterPackFooter()) {
|
||||||
int eof = in.read();
|
int eof = in.read();
|
||||||
if (0 <= eof)
|
if (0 <= eof)
|
||||||
throw new CorruptObjectException(MessageFormat.format(
|
throw new CorruptObjectException(MessageFormat.format(
|
||||||
JGitText.get().expectedEOFReceived,
|
JGitText.get().expectedEOFReceived,
|
||||||
"\\x" + Integer.toHexString(eof)));
|
"\\x" + Integer.toHexString(eof)));
|
||||||
|
} else if (bAvail > 0 && expectDataAfterPackFooter) {
|
||||||
|
in.reset();
|
||||||
|
IO.skipFully(in, bOffset);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!Arrays.equals(actHash, srcHash))
|
if (!Arrays.equals(actHash, srcHash))
|
||||||
|
@ -1142,7 +1162,14 @@ private int fill(final Source src, final int need) throws IOException {
|
||||||
private void sync() throws IOException {
|
private void sync() throws IOException {
|
||||||
packDigest.update(buf, 0, bOffset);
|
packDigest.update(buf, 0, bOffset);
|
||||||
onStoreStream(buf, 0, bOffset);
|
onStoreStream(buf, 0, bOffset);
|
||||||
if (bAvail > 0)
|
if (expectDataAfterPackFooter) {
|
||||||
|
if (bAvail > 0) {
|
||||||
|
in.reset();
|
||||||
|
IO.skipFully(in, bOffset);
|
||||||
|
bAvail = 0;
|
||||||
|
}
|
||||||
|
in.mark(buf.length);
|
||||||
|
} else if (bAvail > 0)
|
||||||
System.arraycopy(buf, bOffset, buf, 0, bAvail);
|
System.arraycopy(buf, bOffset, buf, 0, bAvail);
|
||||||
bBase += bOffset;
|
bBase += bOffset;
|
||||||
bOffset = 0;
|
bOffset = 0;
|
||||||
|
|
Loading…
Reference in New Issue