DfsInserter: Optionally disable existing object check
When using a DfsInserter for high-throughput insertion of many objects (analogous to git-fast-import), we don't necessarily want to do a random object lookup for each. It'll be faster from the inserter's perspective to insert the duplicate objects and let a later GC handle the deduplication. Change-Id: Ic97f5f01657b4525f157e6df66023f1f07fc1851
This commit is contained in:
parent
525baa1213
commit
0d6ba84065
|
@ -219,6 +219,37 @@ public void testInserterIgnoresUnreachable() throws IOException {
|
||||||
assertTrue(pack_sources.contains(PackSource.INSERT));
|
assertTrue(pack_sources.contains(PackSource.INSERT));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNoCheckExisting() throws IOException {
|
||||||
|
byte[] contents = Constants.encode("foo");
|
||||||
|
ObjectId fooId;
|
||||||
|
try (ObjectInserter ins = db.newObjectInserter()) {
|
||||||
|
fooId = ins.insert(Constants.OBJ_BLOB, contents);
|
||||||
|
ins.flush();
|
||||||
|
}
|
||||||
|
assertEquals(1, db.getObjectDatabase().listPacks().size());
|
||||||
|
|
||||||
|
try (ObjectInserter ins = db.newObjectInserter()) {
|
||||||
|
((DfsInserter) ins).checkExisting(false);
|
||||||
|
assertEquals(fooId, ins.insert(Constants.OBJ_BLOB, contents));
|
||||||
|
ins.flush();
|
||||||
|
}
|
||||||
|
assertEquals(2, db.getObjectDatabase().listPacks().size());
|
||||||
|
|
||||||
|
// Verify that we have a foo in both INSERT packs.
|
||||||
|
DfsReader reader = new DfsReader(db.getObjectDatabase());
|
||||||
|
DfsPackFile packs[] = db.getObjectDatabase().getPacks();
|
||||||
|
|
||||||
|
assertEquals(2, packs.length);
|
||||||
|
DfsPackFile p1 = packs[0];
|
||||||
|
assertEquals(PackSource.INSERT, p1.getPackDescription().getPackSource());
|
||||||
|
assertTrue(p1.hasObject(reader, fooId));
|
||||||
|
|
||||||
|
DfsPackFile p2 = packs[1];
|
||||||
|
assertEquals(PackSource.INSERT, p2.getPackDescription().getPackSource());
|
||||||
|
assertTrue(p2.hasObject(reader, fooId));
|
||||||
|
}
|
||||||
|
|
||||||
private static String readString(ObjectLoader loader) throws IOException {
|
private static String readString(ObjectLoader loader) throws IOException {
|
||||||
return RawParseUtils.decode(readStream(loader));
|
return RawParseUtils.decode(readStream(loader));
|
||||||
}
|
}
|
||||||
|
|
|
@ -106,6 +106,7 @@ public class DfsInserter extends ObjectInserter {
|
||||||
DfsPackDescription packDsc;
|
DfsPackDescription packDsc;
|
||||||
PackStream packOut;
|
PackStream packOut;
|
||||||
private boolean rollback;
|
private boolean rollback;
|
||||||
|
private boolean checkExisting = true;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize a new inserter.
|
* Initialize a new inserter.
|
||||||
|
@ -117,6 +118,15 @@ protected DfsInserter(DfsObjDatabase db) {
|
||||||
this.db = db;
|
this.db = db;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param check
|
||||||
|
* if false, will write out possibly-duplicate objects without
|
||||||
|
* first checking whether they exist in the repo; default is true.
|
||||||
|
*/
|
||||||
|
public void checkExisting(boolean check) {
|
||||||
|
checkExisting = check;
|
||||||
|
}
|
||||||
|
|
||||||
void setCompressionLevel(int compression) {
|
void setCompressionLevel(int compression) {
|
||||||
this.compression = compression;
|
this.compression = compression;
|
||||||
}
|
}
|
||||||
|
@ -138,7 +148,7 @@ public ObjectId insert(int type, byte[] data, int off, int len)
|
||||||
if (objectMap != null && objectMap.contains(id))
|
if (objectMap != null && objectMap.contains(id))
|
||||||
return id;
|
return id;
|
||||||
// Ignore unreachable (garbage) objects here.
|
// Ignore unreachable (garbage) objects here.
|
||||||
if (db.has(id, true))
|
if (checkExisting && db.has(id, true))
|
||||||
return id;
|
return id;
|
||||||
|
|
||||||
long offset = beginObject(type, len);
|
long offset = beginObject(type, len);
|
||||||
|
|
Loading…
Reference in New Issue