motiejus
/
jgit
1
Fork 0
Code Releases Activity

PackParserTest: Extract pack-writing helper code to its own class 

PackParserTest has code to create a small pack in memory. We can use
it to test also the DfsPackParser.

Extract the helper functions to a new class (InMemoryPack).

Change-Id: I6dea4d6ea2cac2da8201702df92fd09a687e90a6
This commit is contained in:
Ivan Frade 2023-04-18 15:28:33 -07:00
parent cb99ff5bbb
commit afb013b983
3 changed files with 154 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
org.eclipse.jgit.test/BUILD
View File

@ -29,6 +29,7 @@ HELPERS = glob(
"transport/ObjectIdMatcher.java",
"transport/RequestValidatorTestCase.java",
"transport/SpiTransport.java",
"transport/InMemoryPack.java",
"treewalk/filter/AlwaysCloneTreeFilter.java",
"test/resources/SampleDataRepositoryTestCase.java",
"util/CPUTimeStopWatch.java",

88
org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/InMemoryPack.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Copyright (C) 2023, Google LLC. and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.transport;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.zip.Deflater;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.TemporaryBuffer;
import org.eclipse.jgit.util.TemporaryBuffer.Heap;
/**
* Helper class to create packs for tests.
*/
public class InMemoryPack {
private final Heap tinyPack;
public InMemoryPack() {
this(1024);
}
public InMemoryPack(int size) {
this.tinyPack = new TemporaryBuffer.Heap(size);
}
public InMemoryPack header(int cnt)
throws IOException {
final byte[] hdr = new byte[8];
NB.encodeInt32(hdr, 0, 2);
NB.encodeInt32(hdr, 4, cnt);
tinyPack.write(Constants.PACK_SIGNATURE);
tinyPack.write(hdr, 0, 8);
return this;
}
public InMemoryPack write(int i) throws IOException {
tinyPack.write(i);
return this;
}
public InMemoryPack deflate(byte[] content)
throws IOException {
Deflater deflater = new Deflater();
byte[] buf = new byte[128];
deflater.setInput(content, 0, content.length);
deflater.finish();
do {
final int n = deflater.deflate(buf, 0, buf.length);
if (n > 0)
tinyPack.write(buf, 0, n);
} while (!deflater.finished());
return this;
}
public InMemoryPack copyRaw(AnyObjectId o) throws IOException {
o.copyRawTo(tinyPack);
return this;
}
public InMemoryPack digest() throws IOException {
MessageDigest md = Constants.newMessageDigest();
md.update(tinyPack.toByteArray());
tinyPack.write(md.digest());
return this;
}
public InputStream toInputStream() throws IOException {
return new ByteArrayInputStream(tinyPack.toByteArray());
}
public byte[] toByteArray() throws IOException {
return tinyPack.toByteArray();
}
}

169
org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java
View File

@ -22,12 +22,10 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.Deflater;
import org.eclipse.jgit.errors.TooLargeObjectInPackException;
import org.eclipse.jgit.internal.JGitText;
@ -42,8 +40,6 @@
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevBlob;
import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.TemporaryBuffer;
import org.eclipse.jgit.util.io.UnionInputStream;
import org.junit.After;
import org.junit.Test;
@ -193,17 +189,14 @@ public void testTinyThinPack() throws Exception {
a = d.blob("a");
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
a.copyRawTo(pack);
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
pack.copyRaw(a);
pack.deflate(new byte[] { 0x1, 0x1, 0x1, 'b' });
pack.digest();
digest(pack);
PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
PackParser p = index(pack.toInputStream());
p.setAllowThin(true);
p.parse(NullProgressMonitor.INSTANCE);
}
@ -216,14 +209,14 @@ public void testPackWithDuplicateBlob() throws Exception {
assertTrue(db.getObjectDatabase().has(d.blob(data)));
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_BLOB) << 4 | 0x80 | 1);
pack.write(1);
deflate(pack, data);
digest(pack);
pack.deflate(data);
pack.digest();
PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
PackParser p = index(pack.toInputStream());
p.setAllowThin(false);
p.parse(NullProgressMonitor.INSTANCE);
}
@ -236,16 +229,16 @@ public void testParseOfsDeltaFullSize() throws Exception {
assertTrue(db.getObjectDatabase().has(d.blob(data)));
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 2);
InMemoryPack pack = new InMemoryPack();
pack.header(2);
pack.write((Constants.OBJ_BLOB) << 4 | 10); // offset 12
deflate(pack, data);
pack.deflate(data);
pack.write((Constants.OBJ_OFS_DELTA) << 4 | 4); // offset 31
pack.write(19);
deflate(pack, new byte[] { 0xA, 0xB, 0x1, 'b' });
digest(pack);
pack.deflate(new byte[] { 0xA, 0xB, 0x1, 'b' });
pack.digest();
PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
PackParser p = index(pack.toInputStream());
p.parse(NullProgressMonitor.INSTANCE);
List<PackedObjectInfo> sortedObjectList = p.getSortedObjectList(null);
@ -275,15 +268,15 @@ public void testPackWithTrailingGarbage() throws Exception {
a = d.blob("a");
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
a.copyRawTo(pack);
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
digest(pack);
pack.copyRaw(a);
pack.deflate(new byte[] { 0x1, 0x1, 0x1, 'b' });
pack.digest();
PackParser p = index(new UnionInputStream(
new ByteArrayInputStream(pack.toByteArray()),
pack.toInputStream(),
new ByteArrayInputStream(new byte[] { 0x7e })));
p.setAllowThin(true);
p.setCheckEofAfterPackFooter(true);
@ -305,22 +298,21 @@ public void testMaxObjectSizeFullBlob() throws Exception {
d.blob(data);
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_BLOB) << 4 | 10);
deflate(pack, data);
digest(pack);
pack.deflate(data);
pack.digest();
PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
PackParser p = index(pack.toInputStream());
p.setMaxObjectSizeLimit(11);
p.parse(NullProgressMonitor.INSTANCE);
p = index(new ByteArrayInputStream(pack.toByteArray()));
p = index(pack.toInputStream());
p.setMaxObjectSizeLimit(10);
p.parse(NullProgressMonitor.INSTANCE);
p = index(new ByteArrayInputStream(pack.toByteArray()));
p = index(pack.toInputStream());
p.setMaxObjectSizeLimit(9);
try {
p.parse(NullProgressMonitor.INSTANCE);
@ -339,21 +331,20 @@ public void testMaxObjectSizeDeltaBlock() throws Exception {
a = d.blob("a");
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_REF_DELTA) << 4 | 14);
a.copyRawTo(pack);
deflate(pack, new byte[] { 1, 11, 11, 'a', '0', '1', '2', '3', '4',
pack.copyRaw(a);
pack.deflate(new byte[] { 1, 11, 11, 'a', '0', '1', '2', '3', '4',
'5', '6', '7', '8', '9' });
digest(pack);
pack.digest();
PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
PackParser p = index(pack.toInputStream());
p.setAllowThin(true);
p.setMaxObjectSizeLimit(14);
p.parse(NullProgressMonitor.INSTANCE);
p = index(new ByteArrayInputStream(pack.toByteArray()));
p = index(pack.toInputStream());
p.setAllowThin(true);
p.setMaxObjectSizeLimit(13);
try {
@ -373,20 +364,19 @@ public void testMaxObjectSizeDeltaResultSize() throws Exception {
a = d.blob("0123456789");
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
a.copyRawTo(pack);
deflate(pack, new byte[] { 10, 11, 1, 'a' });
digest(pack);
pack.copyRaw(a);
pack.deflate(new byte[] { 10, 11, 1, 'a' });
pack.digest();
PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
PackParser p = index(pack.toInputStream());
p.setAllowThin(true);
p.setMaxObjectSizeLimit(11);
p.parse(NullProgressMonitor.INSTANCE);
p = index(new ByteArrayInputStream(pack.toByteArray()));
p = index(pack.toInputStream());
p.setAllowThin(true);
p.setMaxObjectSizeLimit(10);
try {
@ -406,12 +396,12 @@ public void testNonMarkingInputStream() throws Exception {
a = d.blob("a");
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
a.copyRawTo(pack);
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
digest(pack);
pack.copyRaw(a);
pack.deflate(new byte[] { 0x1, 0x1, 0x1, 'b' });
pack.digest();
InputStream in = new ByteArrayInputStream(pack.toByteArray()) {
@Override
@ -447,12 +437,12 @@ public void testDataAfterPackFooterSingleRead() throws Exception {
a = d.blob("a");
}
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32*1024);
packHeader(pack, 1);
InMemoryPack pack = new InMemoryPack();
pack.header(1);
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
a.copyRawTo(pack);
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
digest(pack);
pack.copyRaw(a);
pack.deflate(new byte[] { 0x1, 0x1, 0x1, 'b' });
pack.digest();
byte packData[] = pack.toByteArray();
byte streamData[] = new byte[packData.length + 1];
@ -476,14 +466,14 @@ public void testDataAfterPackFooterSplitObjectRead() throws Exception {
// Build a pack ~17k
int objects = 900;
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
packHeader(pack, objects);
InMemoryPack pack = new InMemoryPack(32 * 1024);
pack.header(objects);
for (int i = 0; i < objects; i++) {
pack.write((Constants.OBJ_BLOB) << 4 | 10);
deflate(pack, data);
pack.deflate(data);
}
digest(pack);
pack.digest();
byte packData[] = pack.toByteArray();
byte streamData[] = new byte[packData.length + 1];
@ -510,8 +500,9 @@ public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
}
int objects = 248;
TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
packHeader(pack, objects + 1);
InMemoryPack pack = new InMemoryPack(32 * 1024);
pack.header(objects + 1);
int offset = 13;
StringBuilder sb = new StringBuilder();
for (int i = 0; i < offset; i++)
@ -529,16 +520,16 @@ public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
lenByte |= 1 << 7;
pack.write(lenByte);
}
deflate(pack, Constants.encode(sb.toString()));
pack.deflate(Constants.encode(sb.toString()));
for (int i = 0; i < objects; i++) {
// The last pack header written falls across the 8192 byte boundary
// between [8189:8210]
pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
b.copyRawTo(pack);
deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
pack.copyRaw(b);
pack.deflate(new byte[] { 0x1, 0x1, 0x1, 'b' });
}
digest(pack);
pack.digest();
byte packData[] = pack.toByteArray();
byte streamData[] = new byte[packData.length + 1];
@ -555,36 +546,6 @@ public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
assertEquals(0x7e, in.read());
}
private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
throws IOException {
final byte[] hdr = new byte[8];
NB.encodeInt32(hdr, 0, 2);
NB.encodeInt32(hdr, 4, cnt);
tinyPack.write(Constants.PACK_SIGNATURE);
tinyPack.write(hdr, 0, 8);
}
private static void deflate(TemporaryBuffer.Heap tinyPack,
final byte[] content)
throws IOException {
final Deflater deflater = new Deflater();
final byte[] buf = new byte[128];
deflater.setInput(content, 0, content.length);
deflater.finish();
do {
final int n = deflater.deflate(buf, 0, buf.length);
if (n > 0)
tinyPack.write(buf, 0, n);
} while (!deflater.finished());
}
private static void digest(TemporaryBuffer.Heap buf) throws IOException {
MessageDigest md = Constants.newMessageDigest();
md.update(buf.toByteArray());
buf.write(md.digest());
}
private ObjectInserter inserter;
@After