TemporaryBuffer: Allow presizing block pointer list
Callers may wish to use TemporaryBuffer as an essentially unbounded buffer by passing Integer.MAX_VALUE as the size. (This makes it behave like ByteArrayOutputStream, only without requiring contiguous memory.) Unfortunately, it was always allocating an array in the backing block pointer list to hold enough blocks to MAX_VALUE--all 262,016 of them. It wasn't allocating the blocks themselves, but this array was still extremely wasteful, using about 2MiB of memory on a 64-bit system. Tweak the interface to specify an estimated size, and only allocate the block pointer list enough entries to hold that size. It's an ArrayList, so if that estimate was wrong, it'll grow. We assume the cost of finding enough contiguous memory to grow that array is acceptable. While we're in there, fix an off-by-one error: due to integer division we were undercounting the number of blocks needed to store n bytes of data as (n / SZ). Change-Id: I794eca3ac4472bcc605b3641e177922aca92b9c0
This commit is contained in:
parent
89b91ad406
commit
edf4368b0c
|
@ -53,7 +53,9 @@
|
|||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.eclipse.jgit.junit.TestRng;
|
||||
import org.eclipse.jgit.util.TemporaryBuffer.Block;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TemporaryBufferTest {
|
||||
|
@ -424,4 +426,27 @@ public void testHeap() throws IOException {
|
|||
assertEquals("In-memory buffer limit exceeded", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHeapWithEstimatedSize() throws IOException {
|
||||
int sz = 2 * Block.SZ;
|
||||
try (TemporaryBuffer b = new TemporaryBuffer.Heap(sz / 2, sz)) {
|
||||
for (int i = 0; i < sz; i++) {
|
||||
b.write('x');
|
||||
}
|
||||
try {
|
||||
b.write(1);
|
||||
fail("accepted too many bytes of data");
|
||||
} catch (IOException e) {
|
||||
assertEquals("In-memory buffer limit exceeded", e.getMessage());
|
||||
}
|
||||
|
||||
try (InputStream in = b.openInputStream()) {
|
||||
for (int i = 0; i < sz; i++) {
|
||||
assertEquals('x', in.read());
|
||||
}
|
||||
assertEquals(-1, in.read());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -78,6 +78,9 @@ public abstract class TemporaryBuffer extends OutputStream {
|
|||
*/
|
||||
private int inCoreLimit;
|
||||
|
||||
/** Initial size of block list. */
|
||||
private int initialBlocks;
|
||||
|
||||
/** If {@link #inCoreLimit} has been reached, remainder goes here. */
|
||||
private OutputStream overflow;
|
||||
|
||||
|
@ -86,10 +89,28 @@ public abstract class TemporaryBuffer extends OutputStream {
|
|||
*
|
||||
* @param limit
|
||||
* maximum number of bytes to store in memory before entering the
|
||||
* overflow output path.
|
||||
* overflow output path; also used as the estimated size.
|
||||
*/
|
||||
protected TemporaryBuffer(final int limit) {
|
||||
inCoreLimit = limit;
|
||||
this(limit, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new empty temporary buffer.
|
||||
*
|
||||
* @param estimatedSize
|
||||
* estimated size of storage used, to size the initial list of
|
||||
* block pointers.
|
||||
* @param limit
|
||||
* maximum number of bytes to store in memory before entering the
|
||||
* overflow output path.
|
||||
* @since 4.0
|
||||
*/
|
||||
protected TemporaryBuffer(final int estimatedSize, final int limit) {
|
||||
if (estimatedSize > limit)
|
||||
throw new IllegalArgumentException();
|
||||
this.inCoreLimit = limit;
|
||||
this.initialBlocks = (estimatedSize - 1) / Block.SZ + 1;
|
||||
reset();
|
||||
}
|
||||
|
||||
|
@ -274,7 +295,7 @@ public void reset() {
|
|||
blocks = new ArrayList<Block>(1);
|
||||
blocks.add(new Block(inCoreLimit));
|
||||
} else {
|
||||
blocks = new ArrayList<Block>(inCoreLimit / Block.SZ);
|
||||
blocks = new ArrayList<Block>(initialBlocks);
|
||||
blocks.add(new Block());
|
||||
}
|
||||
}
|
||||
|
@ -498,14 +519,30 @@ public static class Heap extends TemporaryBuffer {
|
|||
* Create a new heap buffer with a maximum storage limit.
|
||||
*
|
||||
* @param limit
|
||||
* maximum number of bytes that can be stored in this buffer.
|
||||
* Storing beyond this many will cause an IOException to be
|
||||
* thrown during write.
|
||||
* maximum number of bytes that can be stored in this buffer;
|
||||
* also used as the estimated size. Storing beyond this many
|
||||
* will cause an IOException to be thrown during write.
|
||||
*/
|
||||
public Heap(final int limit) {
|
||||
super(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new heap buffer with a maximum storage limit.
|
||||
*
|
||||
* @param estimatedSize
|
||||
* estimated size of storage used, to size the initial list of
|
||||
* block pointers.
|
||||
* @param limit
|
||||
* maximum number of bytes that can be stored in this buffer.
|
||||
* Storing beyond this many will cause an IOException to be
|
||||
* thrown during write.
|
||||
* @since 4.0
|
||||
*/
|
||||
public Heap(final int estimatedSize, final int limit) {
|
||||
super(estimatedSize, limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected OutputStream overflow() throws IOException {
|
||||
throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
|
||||
|
|
Loading…
Reference in New Issue