reftable: pass OutputStream at construction time

This makes the intended use of the classes more clear. It also
simplifies generic functions that write reftables: they only need a
ReftableWriter as argument, as the stream is carried within the
ReftableWriter.

Change-Id: Idbb06f89ae33100f0c0b562cc38e5b3b026d5181
Signed-off-by: Han-Wen Nienhuys <hanwen@google.com>
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
This commit is contained in:
Han-Wen Nienhuys 2019-09-18 15:18:51 +02:00 committed by Matthias Sohn
parent d75a6b5d81
commit e5880d9531
11 changed files with 125 additions and 146 deletions

View File

@ -128,9 +128,9 @@ protected void run() throws Exception {
cfg.setMaxIndexLevels(indexLevels); cfg.setMaxIndexLevels(indexLevels);
} }
ReftableWriter w = new ReftableWriter(cfg); ReftableWriter w = new ReftableWriter(cfg, os);
w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs)); w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
w.begin(os); w.begin();
w.sortAndWriteRefs(refs); w.sortAndWriteRefs(refs);
for (LogEntry e : logs) { for (LogEntry e : logs) {
w.writeLog(e.ref, e.updateIndex, e.who, w.writeLog(e.ref, e.updateIndex, e.who,

View File

@ -723,7 +723,7 @@ public void leavesNonGcReftablesIfNotConfigured() throws Exception {
DfsPackDescription t1 = odb.newPack(INSERT); DfsPackDescription t1 = odb.newPack(INSERT);
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
new ReftableWriter().begin(out).finish(); new ReftableWriter(out).begin().finish();
t1.addFileExt(REFTABLE); t1.addFileExt(REFTABLE);
} }
odb.commitPack(Collections.singleton(t1), null); odb.commitPack(Collections.singleton(t1), null);
@ -755,7 +755,7 @@ public void prunesNonGcReftables() throws Exception {
DfsPackDescription t1 = odb.newPack(INSERT); DfsPackDescription t1 = odb.newPack(INSERT);
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
new ReftableWriter().begin(out).finish(); new ReftableWriter(out).begin().finish();
t1.addFileExt(REFTABLE); t1.addFileExt(REFTABLE);
} }
odb.commitPack(Collections.singleton(t1), null); odb.commitPack(Collections.singleton(t1), null);
@ -795,10 +795,10 @@ public void compactsReftables() throws Exception {
Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
"refs/heads/next", commit0.copy()); "refs/heads/next", commit0.copy());
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
ReftableWriter w = new ReftableWriter(); ReftableWriter w = new ReftableWriter(out);
w.setMinUpdateIndex(42); w.setMinUpdateIndex(42);
w.setMaxUpdateIndex(42); w.setMaxUpdateIndex(42);
w.begin(out); w.begin();
w.sortAndWriteRefs(Collections.singleton(next)); w.sortAndWriteRefs(Collections.singleton(next));
w.finish(); w.finish();
t1.addFileExt(REFTABLE); t1.addFileExt(REFTABLE);
@ -877,10 +877,10 @@ public void reftableWithoutTombstoneResurrected() throws Exception {
Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT, Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT,
commit1); commit1);
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
ReftableWriter w = new ReftableWriter(); ReftableWriter w = new ReftableWriter(out)
w.setMinUpdateIndex(1); .setMinUpdateIndex(1)
w.setMaxUpdateIndex(1); .setMaxUpdateIndex(1)
w.begin(out); .begin();
w.writeRef(newNext, 1); w.writeRef(newNext, 1);
w.finish(); w.finish();
t1.addFileExt(REFTABLE); t1.addFileExt(REFTABLE);
@ -929,10 +929,10 @@ public void reftableWithTombstoneNotResurrected() throws Exception {
Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT, Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT,
commit1); commit1);
try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
ReftableWriter w = new ReftableWriter(); ReftableWriter w = new ReftableWriter(out)
w.setMinUpdateIndex(1); .setMinUpdateIndex(1)
w.setMaxUpdateIndex(1); .setMaxUpdateIndex(1)
w.begin(out); .begin();
w.writeRef(newNext, 1); w.writeRef(newNext, 1);
w.finish(); w.finish();
t1.addFileExt(REFTABLE); t1.addFileExt(REFTABLE);

View File

@ -297,10 +297,10 @@ public void oneTableSeek() throws IOException {
@Test @Test
public void missedUpdate() throws IOException { public void missedUpdate() throws IOException {
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(buf)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(3) .setMaxUpdateIndex(3)
.begin(buf); .begin();
writer.writeRef(ref("refs/heads/a", 1), 1); writer.writeRef(ref("refs/heads/a", 1), 1);
writer.writeRef(ref("refs/heads/c", 3), 3); writer.writeRef(ref("refs/heads/c", 3), 3);
writer.finish(); writer.finish();
@ -337,13 +337,13 @@ public void compaction() throws IOException {
List<Ref> delta2 = Arrays.asList(delete("refs/heads/next")); List<Ref> delta2 = Arrays.asList(delete("refs/heads/next"));
List<Ref> delta3 = Arrays.asList(ref("refs/heads/master", 8)); List<Ref> delta3 = Arrays.asList(ref("refs/heads/master", 8));
ReftableCompactor compactor = new ReftableCompactor(); ByteArrayOutputStream out = new ByteArrayOutputStream();
ReftableCompactor compactor = new ReftableCompactor(out);
compactor.addAll(Arrays.asList( compactor.addAll(Arrays.asList(
read(write(delta1)), read(write(delta1)),
read(write(delta2)), read(write(delta2)),
read(write(delta3)))); read(write(delta3))));
ByteArrayOutputStream out = new ByteArrayOutputStream(); compactor.compact();
compactor.compact(out);
byte[] table = out.toByteArray(); byte[] table = out.toByteArray();
ReftableReader reader = read(table); ReftableReader reader = read(table);
@ -461,10 +461,10 @@ private byte[] write(Collection<Ref> refs) throws IOException {
private byte[] write(Collection<Ref> refs, long updateIndex) private byte[] write(Collection<Ref> refs, long updateIndex)
throws IOException { throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
new ReftableWriter() new ReftableWriter(buffer)
.setMinUpdateIndex(updateIndex) .setMinUpdateIndex(updateIndex)
.setMaxUpdateIndex(updateIndex) .setMaxUpdateIndex(updateIndex)
.begin(buffer) .begin()
.sortAndWriteRefs(refs) .sortAndWriteRefs(refs)
.finish(); .finish();
return buffer.toByteArray(); return buffer.toByteArray();

View File

@ -67,9 +67,10 @@ public class ReftableCompactorTest {
@Test @Test
public void noTables() throws IOException { public void noTables() throws IOException {
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor;
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
compactor.compact(out); compactor = new ReftableCompactor(out);
compactor.compact();
} }
Stats stats = compactor.getStats(); Stats stats = compactor.getStats();
assertEquals(0, stats.minUpdateIndex()); assertEquals(0, stats.minUpdateIndex());
@ -81,10 +82,10 @@ public void noTables() throws IOException {
public void oneTable() throws IOException { public void oneTable() throws IOException {
byte[] inTab; byte[] inTab;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(0) .setMinUpdateIndex(0)
.setMaxUpdateIndex(0) .setMaxUpdateIndex(0)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 1)); writer.writeRef(ref(MASTER, 1));
writer.finish(); writer.finish();
@ -92,10 +93,11 @@ public void oneTable() throws IOException {
} }
byte[] outTab; byte[] outTab;
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor;
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
compactor = new ReftableCompactor(outBuf);
compactor.tryAddFirst(read(inTab)); compactor.tryAddFirst(read(inTab));
compactor.compact(outBuf); compactor.compact();
outTab = outBuf.toByteArray(); outTab = outBuf.toByteArray();
} }
Stats stats = compactor.getStats(); Stats stats = compactor.getStats();
@ -116,10 +118,10 @@ public void oneTable() throws IOException {
public void twoTablesOneRef() throws IOException { public void twoTablesOneRef() throws IOException {
byte[] inTab1; byte[] inTab1;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(0) .setMinUpdateIndex(0)
.setMaxUpdateIndex(0) .setMaxUpdateIndex(0)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 1)); writer.writeRef(ref(MASTER, 1));
writer.finish(); writer.finish();
@ -128,10 +130,10 @@ public void twoTablesOneRef() throws IOException {
byte[] inTab2; byte[] inTab2;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 2)); writer.writeRef(ref(MASTER, 2));
writer.finish(); writer.finish();
@ -139,10 +141,11 @@ public void twoTablesOneRef() throws IOException {
} }
byte[] outTab; byte[] outTab;
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor;
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
compactor = new ReftableCompactor(outBuf);
compactor.addAll(Arrays.asList(read(inTab1), read(inTab2))); compactor.addAll(Arrays.asList(read(inTab1), read(inTab2)));
compactor.compact(outBuf); compactor.compact();
outTab = outBuf.toByteArray(); outTab = outBuf.toByteArray();
} }
Stats stats = compactor.getStats(); Stats stats = compactor.getStats();
@ -163,10 +166,10 @@ public void twoTablesOneRef() throws IOException {
public void twoTablesTwoRefs() throws IOException { public void twoTablesTwoRefs() throws IOException {
byte[] inTab1; byte[] inTab1;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(0) .setMinUpdateIndex(0)
.setMaxUpdateIndex(0) .setMaxUpdateIndex(0)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 1)); writer.writeRef(ref(MASTER, 1));
writer.writeRef(ref(NEXT, 2)); writer.writeRef(ref(NEXT, 2));
@ -176,10 +179,10 @@ public void twoTablesTwoRefs() throws IOException {
byte[] inTab2; byte[] inTab2;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 3)); writer.writeRef(ref(MASTER, 3));
writer.finish(); writer.finish();
@ -187,10 +190,11 @@ public void twoTablesTwoRefs() throws IOException {
} }
byte[] outTab; byte[] outTab;
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor;
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
compactor = new ReftableCompactor(outBuf);
compactor.addAll(Arrays.asList(read(inTab1), read(inTab2))); compactor.addAll(Arrays.asList(read(inTab1), read(inTab2)));
compactor.compact(outBuf); compactor.compact();
outTab = outBuf.toByteArray(); outTab = outBuf.toByteArray();
} }
Stats stats = compactor.getStats(); Stats stats = compactor.getStats();
@ -216,10 +220,10 @@ public void twoTablesTwoRefs() throws IOException {
public void twoTablesIncludeOneDelete() throws IOException { public void twoTablesIncludeOneDelete() throws IOException {
byte[] inTab1; byte[] inTab1;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(0) .setMinUpdateIndex(0)
.setMaxUpdateIndex(0) .setMaxUpdateIndex(0)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 1)); writer.writeRef(ref(MASTER, 1));
writer.finish(); writer.finish();
@ -228,10 +232,10 @@ public void twoTablesIncludeOneDelete() throws IOException {
byte[] inTab2; byte[] inTab2;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(inBuf); .begin();
writer.writeRef(tombstone(MASTER)); writer.writeRef(tombstone(MASTER));
writer.finish(); writer.finish();
@ -239,11 +243,12 @@ public void twoTablesIncludeOneDelete() throws IOException {
} }
byte[] outTab; byte[] outTab;
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor;
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
compactor = new ReftableCompactor(outBuf);
compactor.setIncludeDeletes(true); compactor.setIncludeDeletes(true);
compactor.addAll(Arrays.asList(read(inTab1), read(inTab2))); compactor.addAll(Arrays.asList(read(inTab1), read(inTab2)));
compactor.compact(outBuf); compactor.compact();
outTab = outBuf.toByteArray(); outTab = outBuf.toByteArray();
} }
Stats stats = compactor.getStats(); Stats stats = compactor.getStats();
@ -261,10 +266,10 @@ public void twoTablesIncludeOneDelete() throws IOException {
public void twoTablesNotIncludeOneDelete() throws IOException { public void twoTablesNotIncludeOneDelete() throws IOException {
byte[] inTab1; byte[] inTab1;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(0) .setMinUpdateIndex(0)
.setMaxUpdateIndex(0) .setMaxUpdateIndex(0)
.begin(inBuf); .begin();
writer.writeRef(ref(MASTER, 1)); writer.writeRef(ref(MASTER, 1));
writer.finish(); writer.finish();
@ -273,10 +278,10 @@ public void twoTablesNotIncludeOneDelete() throws IOException {
byte[] inTab2; byte[] inTab2;
try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream inBuf = new ByteArrayOutputStream()) {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(inBuf)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(inBuf); .begin();
writer.writeRef(tombstone(MASTER)); writer.writeRef(tombstone(MASTER));
writer.finish(); writer.finish();
@ -284,11 +289,12 @@ public void twoTablesNotIncludeOneDelete() throws IOException {
} }
byte[] outTab; byte[] outTab;
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor;
try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream outBuf = new ByteArrayOutputStream()) {
compactor = new ReftableCompactor(outBuf);
compactor.setIncludeDeletes(false); compactor.setIncludeDeletes(false);
compactor.addAll(Arrays.asList(read(inTab1), read(inTab2))); compactor.addAll(Arrays.asList(read(inTab1), read(inTab2)));
compactor.compact(outBuf); compactor.compact();
outTab = outBuf.toByteArray(); outTab = outBuf.toByteArray();
} }
Stats stats = compactor.getStats(); Stats stats = compactor.getStats();

View File

@ -137,9 +137,9 @@ public void estimateCurrentBytesOneRef() throws IOException {
byte[] table; byte[] table;
ReftableConfig cfg = new ReftableConfig(); ReftableConfig cfg = new ReftableConfig();
cfg.setIndexObjects(false); cfg.setIndexObjects(false);
ReftableWriter writer = new ReftableWriter().setConfig(cfg);
try (ByteArrayOutputStream buf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream buf = new ByteArrayOutputStream()) {
writer.begin(buf); ReftableWriter writer = new ReftableWriter(buf).setConfig(cfg);
writer.begin();
assertEquals(92, writer.estimateTotalBytes()); assertEquals(92, writer.estimateTotalBytes());
writer.writeRef(exp); writer.writeRef(exp);
assertEquals(expBytes, writer.estimateTotalBytes()); assertEquals(expBytes, writer.estimateTotalBytes());
@ -163,9 +163,9 @@ public void estimateCurrentBytesWithIndex() throws IOException {
int expBytes = 147860; int expBytes = 147860;
byte[] table; byte[] table;
ReftableWriter writer = new ReftableWriter().setConfig(cfg);
try (ByteArrayOutputStream buf = new ByteArrayOutputStream()) { try (ByteArrayOutputStream buf = new ByteArrayOutputStream()) {
writer.begin(buf); ReftableWriter writer = new ReftableWriter(buf).setConfig(cfg);
writer.begin();
writer.sortAndWriteRefs(refs); writer.sortAndWriteRefs(refs);
assertEquals(expBytes, writer.estimateTotalBytes()); assertEquals(expBytes, writer.estimateTotalBytes());
writer.finish(); writer.finish();
@ -424,10 +424,10 @@ public void noIndexSeek() throws IOException {
public void invalidRefWriteOrder() throws IOException { public void invalidRefWriteOrder() throws IOException {
Ref master = ref(MASTER, 1); Ref master = ref(MASTER, 1);
Ref next = ref(NEXT, 2); Ref next = ref(NEXT, 2);
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(new ByteArrayOutputStream())
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(new ByteArrayOutputStream()); .begin();
writer.writeRef(next); writer.writeRef(next);
IllegalArgumentException e = assertThrows( IllegalArgumentException e = assertThrows(
@ -438,10 +438,10 @@ public void invalidRefWriteOrder() throws IOException {
@Test @Test
public void invalidReflogWriteOrderUpdateIndex() throws IOException { public void invalidReflogWriteOrderUpdateIndex() throws IOException {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(new ByteArrayOutputStream())
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(2) .setMaxUpdateIndex(2)
.begin(new ByteArrayOutputStream()); .begin();
PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60); PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60);
String msg = "test"; String msg = "test";
@ -454,10 +454,10 @@ public void invalidReflogWriteOrderUpdateIndex() throws IOException {
@Test @Test
public void invalidReflogWriteOrderName() throws IOException { public void invalidReflogWriteOrderName() throws IOException {
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(new ByteArrayOutputStream())
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(new ByteArrayOutputStream()); .begin();
PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60); PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60);
String msg = "test"; String msg = "test";
@ -476,10 +476,10 @@ public void withReflog() throws IOException {
String msg = "test"; String msg = "test";
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(buffer)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(buffer); .begin();
writer.writeRef(master); writer.writeRef(master);
writer.writeRef(next); writer.writeRef(next);
@ -531,8 +531,8 @@ public void reflogReader() throws IOException {
Ref next = ref(NEXT, 2); Ref next = ref(NEXT, 2);
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter().setMinUpdateIndex(1) ReftableWriter writer = new ReftableWriter(buffer).setMinUpdateIndex(1)
.setMaxUpdateIndex(1).begin(buffer); .setMaxUpdateIndex(1).begin();
writer.writeRef(master); writer.writeRef(master);
writer.writeRef(next); writer.writeRef(next);
@ -573,11 +573,11 @@ public void allRefs() throws IOException {
cfg.setRefBlockSize(1024); cfg.setRefBlockSize(1024);
cfg.setLogBlockSize(1024); cfg.setLogBlockSize(1024);
cfg.setAlignBlocks(true); cfg.setAlignBlocks(true);
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(buffer)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.setConfig(cfg) .setConfig(cfg)
.begin(buffer); .begin();
PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60); PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60);
// Fill out the 1st ref block. // Fill out the 1st ref block.
@ -611,10 +611,10 @@ public void reflogSeek() throws IOException {
String msgNext = "test next"; String msgNext = "test next";
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(buffer)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(buffer); .begin();
writer.writeLog(MASTER, 1, who, ObjectId.zeroId(), id(1), msg); writer.writeLog(MASTER, 1, who, ObjectId.zeroId(), id(1), msg);
writer.writeLog(NEXT, 1, who, ObjectId.zeroId(), id(2), msgNext); writer.writeLog(NEXT, 1, who, ObjectId.zeroId(), id(2), msgNext);
@ -654,10 +654,10 @@ public void reflogSeekPrefix() throws IOException {
PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60); PersonIdent who = new PersonIdent("Log", "Ger", 1500079709, -8 * 60);
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(buffer)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(buffer); .begin();
writer.writeLog("branchname", 1, who, ObjectId.zeroId(), id(1), "branchname"); writer.writeLog("branchname", 1, who, ObjectId.zeroId(), id(1), "branchname");
@ -678,10 +678,10 @@ public void onlyReflog() throws IOException {
String msg = "test"; String msg = "test";
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter() ReftableWriter writer = new ReftableWriter(buffer)
.setMinUpdateIndex(1) .setMinUpdateIndex(1)
.setMaxUpdateIndex(1) .setMaxUpdateIndex(1)
.begin(buffer); .begin();
writer.writeLog(MASTER, 1, who, ObjectId.zeroId(), id(1), msg); writer.writeLog(MASTER, 1, who, ObjectId.zeroId(), id(1), msg);
writer.writeLog(NEXT, 1, who, ObjectId.zeroId(), id(2), msg); writer.writeLog(NEXT, 1, who, ObjectId.zeroId(), id(2), msg);
writer.finish(); writer.finish();
@ -730,8 +730,8 @@ public void logScan() throws IOException {
cfg.setLogBlockSize(2048); cfg.setLogBlockSize(2048);
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter(cfg); ReftableWriter writer = new ReftableWriter(cfg, buffer);
writer.setMinUpdateIndex(1).setMaxUpdateIndex(1).begin(buffer); writer.setMinUpdateIndex(1).setMaxUpdateIndex(1).begin();
List<Ref> refs = new ArrayList<>(); List<Ref> refs = new ArrayList<>();
for (int i = 1; i <= 5670; i++) { for (int i = 1; i <= 5670; i++) {
@ -850,7 +850,7 @@ public void nameTooLongDoesNotWrite() throws IOException {
cfg.setRefBlockSize(64); cfg.setRefBlockSize(64);
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
ReftableWriter writer = new ReftableWriter(cfg).begin(buffer); ReftableWriter writer = new ReftableWriter(cfg, buffer).begin();
writer.writeRef(ref("refs/heads/i-am-not-a-teapot", 1)); writer.writeRef(ref("refs/heads/i-am-not-a-teapot", 1));
writer.finish(); writer.finish();
fail("expected BlockSizeTooSmallException"); fail("expected BlockSizeTooSmallException");
@ -935,8 +935,8 @@ private byte[] write(Ref... refs) throws IOException {
private byte[] write(Collection<Ref> refs) throws IOException { private byte[] write(Collection<Ref> refs) throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
stats = new ReftableWriter() stats = new ReftableWriter(buffer)
.begin(buffer) .begin()
.sortAndWriteRefs(refs) .sortAndWriteRefs(refs)
.finish() .finish()
.getStats(); .getStats();

View File

@ -744,11 +744,15 @@ private void writeReftable(DfsPackDescription pack) throws IOException {
return; return;
} }
try (DfsReftableStack stack = DfsReftableStack.open(ctx, reftablesBefore)) { try (DfsReftableStack stack = DfsReftableStack.open(ctx, reftablesBefore);
ReftableCompactor compact = new ReftableCompactor(); DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
ReftableCompactor compact = new ReftableCompactor(out);
compact.addAll(stack.readers()); compact.addAll(stack.readers());
compact.setIncludeDeletes(includeDeletes); compact.setIncludeDeletes(includeDeletes);
compactReftable(pack, compact); compact.setConfig(configureReftable(reftableConfig, out));
compact.compact();
pack.addFileExt(REFTABLE);
pack.setReftableStats(compact.getStats());
} }
} }
@ -773,14 +777,4 @@ private void writeReftable(DfsPackDescription pack, Collection<Ref> refs)
pack.setReftableStats(writer.getStats()); pack.setReftableStats(writer.getStats());
} }
} }
private void compactReftable(DfsPackDescription pack,
ReftableCompactor compact) throws IOException {
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
compact.setConfig(configureReftable(reftableConfig, out));
compact.compact(out);
pack.addFileExt(REFTABLE);
pack.setReftableStats(compact.getStats());
}
}
} }

View File

@ -311,12 +311,16 @@ private void compactReftables(DfsReader ctx) throws IOException {
DfsObjDatabase objdb = repo.getObjectDatabase(); DfsObjDatabase objdb = repo.getObjectDatabase();
Collections.sort(srcReftables, objdb.reftableComparator()); Collections.sort(srcReftables, objdb.reftableComparator());
try (DfsReftableStack stack = DfsReftableStack.open(ctx, srcReftables)) { try (DfsReftableStack stack = DfsReftableStack.open(ctx, srcReftables);
DfsOutputStream out = objdb.writeFile(outDesc, REFTABLE)) {
initOutDesc(objdb); initOutDesc(objdb);
ReftableCompactor compact = new ReftableCompactor(); ReftableCompactor compact = new ReftableCompactor(out);
compact.addAll(stack.readers()); compact.addAll(stack.readers());
compact.setIncludeDeletes(true); compact.setIncludeDeletes(true);
writeReftable(objdb, outDesc, compact); compact.setConfig(configureReftable(reftableConfig, out));
compact.compact();
outDesc.addFileExt(REFTABLE);
outDesc.setReftableStats(compact.getStats());
} }
} }
@ -497,16 +501,6 @@ private static void writeIndex(DfsObjDatabase objdb,
} }
} }
private void writeReftable(DfsObjDatabase objdb, DfsPackDescription pack,
ReftableCompactor compact) throws IOException {
try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
compact.setConfig(configureReftable(reftableConfig, out));
compact.compact(out);
pack.addFileExt(REFTABLE);
pack.setReftableStats(compact.getStats());
}
}
static ReftableConfig configureReftable(ReftableConfig cfg, static ReftableConfig configureReftable(ReftableConfig cfg,
DfsOutputStream out) { DfsOutputStream out) {
int bs = out.blockSize(); int bs = out.blockSize();

View File

@ -296,9 +296,9 @@ && canCompactTopOfStack(cfg)) {
private ReftableWriter.Stats write(OutputStream os, ReftableConfig cfg, private ReftableWriter.Stats write(OutputStream os, ReftableConfig cfg,
long updateIndex, List<Ref> newRefs, List<ReceiveCommand> pending) long updateIndex, List<Ref> newRefs, List<ReceiveCommand> pending)
throws IOException { throws IOException {
ReftableWriter writer = new ReftableWriter(cfg) ReftableWriter writer = new ReftableWriter(cfg, os)
.setMinUpdateIndex(updateIndex).setMaxUpdateIndex(updateIndex) .setMinUpdateIndex(updateIndex).setMaxUpdateIndex(updateIndex)
.begin(os).sortAndWriteRefs(newRefs); .begin().sortAndWriteRefs(newRefs);
if (!isRefLogDisabled()) { if (!isRefLogDisabled()) {
writeLog(writer, updateIndex, pending); writeLog(writer, updateIndex, pending);
} }
@ -434,11 +434,11 @@ private ReftableWriter.Stats compactTopOfStack(OutputStream out,
tables.add(last); tables.add(last);
tables.add(new ReftableReader(BlockSource.from(newTable))); tables.add(new ReftableReader(BlockSource.from(newTable)));
ReftableCompactor compactor = new ReftableCompactor(); ReftableCompactor compactor = new ReftableCompactor(out);
compactor.setConfig(cfg); compactor.setConfig(cfg);
compactor.setIncludeDeletes(true); compactor.setIncludeDeletes(true);
compactor.addAll(tables); compactor.addAll(tables);
compactor.compact(out); compactor.compact();
return compactor.getStats(); return compactor.getStats();
} }

View File

@ -72,9 +72,9 @@ public static Reftable from(Collection<Ref> refs) {
cfg.setIndexObjects(false); cfg.setIndexObjects(false);
cfg.setAlignBlocks(false); cfg.setAlignBlocks(false);
ByteArrayOutputStream buf = new ByteArrayOutputStream(); ByteArrayOutputStream buf = new ByteArrayOutputStream();
new ReftableWriter() new ReftableWriter(buf)
.setConfig(cfg) .setConfig(cfg)
.begin(buf) .begin()
.sortAndWriteRefs(refs) .sortAndWriteRefs(refs)
.finish(); .finish();
return new ReftableReader(BlockSource.from(buf.toByteArray())); return new ReftableReader(BlockSource.from(buf.toByteArray()));

View File

@ -68,7 +68,7 @@
* {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}. * {@code setOldestReflogTimeMillis(Long.MAX_VALUE)}.
*/ */
public class ReftableCompactor { public class ReftableCompactor {
private final ReftableWriter writer = new ReftableWriter(); private final ReftableWriter writer;
private final ArrayDeque<Reftable> tables = new ArrayDeque<>(); private final ArrayDeque<Reftable> tables = new ArrayDeque<>();
private long compactBytesLimit; private long compactBytesLimit;
@ -79,6 +79,17 @@ public class ReftableCompactor {
private long oldestReflogTimeMillis; private long oldestReflogTimeMillis;
private Stats stats; private Stats stats;
/**
* Creates a new compactor.
*
* @param out
* stream to write the compacted tables to. Caller is responsible
* for closing {@code out}.
*/
public ReftableCompactor(OutputStream out) {
writer = new ReftableWriter(out);
}
/** /**
* Set configuration for the reftable. * Set configuration for the reftable.
* *
@ -225,19 +236,16 @@ private void adjustUpdateIndexes(ReftableReader reader) throws IOException {
/** /**
* Write a compaction to {@code out}. * Write a compaction to {@code out}.
* *
* @param out
* stream to write the compacted tables to. Caller is responsible
* for closing {@code out}.
* @throws java.io.IOException * @throws java.io.IOException
* if tables cannot be read, or cannot be written. * if tables cannot be read, or cannot be written.
*/ */
public void compact(OutputStream out) throws IOException { public void compact() throws IOException {
MergedReftable mr = new MergedReftable(new ArrayList<>(tables)); MergedReftable mr = new MergedReftable(new ArrayList<>(tables));
mr.setIncludeDeletes(includeDeletes); mr.setIncludeDeletes(includeDeletes);
writer.setMinUpdateIndex(Math.max(minUpdateIndex, 0)); writer.setMinUpdateIndex(Math.max(minUpdateIndex, 0));
writer.setMaxUpdateIndex(maxUpdateIndex); writer.setMaxUpdateIndex(maxUpdateIndex);
writer.begin(out); writer.begin();
mergeRefs(mr); mergeRefs(mr);
mergeLogs(mr); mergeLogs(mr);
writer.finish(); writer.finish();

View File

@ -123,32 +123,25 @@ public class ReftableWriter {
/** /**
* Initialize a writer with a default configuration. * Initialize a writer with a default configuration.
*
* @param os
* output stream.
*/ */
public ReftableWriter() { public ReftableWriter(OutputStream os) {
this(new ReftableConfig()); this(new ReftableConfig(), os);
lastRef = null; lastRef = null;
lastLog = null; lastLog = null;
} }
/**
* Initialize a writer with a specific configuration.
*
* @param cfg
* configuration for the writer.
*/
public ReftableWriter(ReftableConfig cfg) {
this(cfg, null);
}
/** /**
* Initialize a writer with a configuration. * Initialize a writer with a configuration.
* *
* @param cfg * @param cfg
* configuration for the writer * configuration for the writer
* @param os * @param os
* output stream. Do not supply a stream to begin() on this writer. * output stream.
*/ */
public ReftableWriter(ReftableConfig cfg, @Nullable OutputStream os) { public ReftableWriter(ReftableConfig cfg, OutputStream os) {
config = cfg; config = cfg;
outputStream = os; outputStream = os;
} }
@ -196,22 +189,6 @@ public ReftableWriter setMaxUpdateIndex(long max) {
return this; return this;
} }
/**
* Begin writing the reftable. Should be called only once.
*
* @param os
* stream to write the table to. Caller is responsible for
* closing the stream after invoking {@link #finish()}.
* @return {@code this}
*/
public ReftableWriter begin(OutputStream os) {
if (outputStream != null) {
throw new IllegalStateException("begin() called twice.");//$NON-NLS-1$
}
outputStream = os;
return begin();
}
/** /**
* Begin writing the reftable. Should be called only once. Call this * Begin writing the reftable. Should be called only once. Call this
* if a stream was passed to the constructor. * if a stream was passed to the constructor.