DHT: Drop leading hash digits from row keys
Originally I put the first two digits of the object SHA-1 into the start of a row key to try and spread the load of objects around a DHT service. Unfortunately this tends to not work as well as I had hoped. Servers reading a repository need to contact every node in a DHT cluster if the cluster tries to evenly distribute the object rows. This is a lot of connections, especially if the cluster has many backend storage servers. If the library has an open connection limit (possibly due to JVM file descriptor limitations) it may need to open and close a lot of connections to access a repository, rather than being able to reuse the same connection to a handful of backend servers. This results in a lot of connection thrashing for some DHT type databases, and is inefficient. Some DHTs are able to operate even if part of the database space is currently unavailable. For example, a DHT service might assign some section of the key space to a node, and then fail that section over to another node when the primary is noticed as being offline. During that failover period that section of the key space is not available, but other sections hosted by other backends are still ready for service. Spreading keys all over the cluster makes it likely that any single backend being temporarily down means the entire cluster is down, rather than only some. This is a massive schema change, but it should improve relability and performance for any DHT system. Change-Id: I6b65bfb4c14b6f7bd323c2bd0638b49d429245be Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
This commit is contained in:
parent
7ff6eb584c
commit
0e1d5ad8f8
|
@ -43,7 +43,8 @@
|
||||||
|
|
||||||
package org.eclipse.jgit.storage.dht;
|
package org.eclipse.jgit.storage.dht;
|
||||||
|
|
||||||
import static org.junit.Assert.*;
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
|
|
||||||
import org.eclipse.jgit.lib.ObjectId;
|
import org.eclipse.jgit.lib.ObjectId;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -59,19 +60,19 @@ public void testKey() {
|
||||||
ChunkKey key1 = ChunkKey.create(repo1, id);
|
ChunkKey key1 = ChunkKey.create(repo1, id);
|
||||||
assertEquals(repo1.asInt(), key1.getRepositoryId());
|
assertEquals(repo1.asInt(), key1.getRepositoryId());
|
||||||
assertEquals(id, key1.getChunkHash());
|
assertEquals(id, key1.getChunkHash());
|
||||||
assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
||||||
key1.asString());
|
key1.asString());
|
||||||
|
|
||||||
ChunkKey key2 = ChunkKey.fromBytes(key1.asBytes());
|
ChunkKey key2 = ChunkKey.fromBytes(key1.asBytes());
|
||||||
assertEquals(repo1.asInt(), key2.getRepositoryId());
|
assertEquals(repo1.asInt(), key2.getRepositoryId());
|
||||||
assertEquals(id, key2.getChunkHash());
|
assertEquals(id, key2.getChunkHash());
|
||||||
assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
||||||
key2.asString());
|
key2.asString());
|
||||||
|
|
||||||
ChunkKey key3 = ChunkKey.fromString(key1.asString());
|
ChunkKey key3 = ChunkKey.fromString(key1.asString());
|
||||||
assertEquals(repo1.asInt(), key3.getRepositoryId());
|
assertEquals(repo1.asInt(), key3.getRepositoryId());
|
||||||
assertEquals(id, key3.getChunkHash());
|
assertEquals(id, key3.getChunkHash());
|
||||||
assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
||||||
key3.asString());
|
key3.asString());
|
||||||
|
|
||||||
assertEquals(key1, key2);
|
assertEquals(key1, key2);
|
||||||
|
|
|
@ -58,19 +58,19 @@ public void testKey() {
|
||||||
ObjectIndexKey key1 = ObjectIndexKey.create(repo, id);
|
ObjectIndexKey key1 = ObjectIndexKey.create(repo, id);
|
||||||
assertEquals(repo.asInt(), key1.getRepositoryId());
|
assertEquals(repo.asInt(), key1.getRepositoryId());
|
||||||
assertEquals(key1, id);
|
assertEquals(key1, id);
|
||||||
assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
||||||
key1.asString());
|
key1.asString());
|
||||||
|
|
||||||
ObjectIndexKey key2 = ObjectIndexKey.fromBytes(key1.asBytes());
|
ObjectIndexKey key2 = ObjectIndexKey.fromBytes(key1.asBytes());
|
||||||
assertEquals(repo.asInt(), key2.getRepositoryId());
|
assertEquals(repo.asInt(), key2.getRepositoryId());
|
||||||
assertEquals(key2, id);
|
assertEquals(key2, id);
|
||||||
assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
||||||
key2.asString());
|
key2.asString());
|
||||||
|
|
||||||
ObjectIndexKey key3 = ObjectIndexKey.fromString(key1.asString());
|
ObjectIndexKey key3 = ObjectIndexKey.fromString(key1.asString());
|
||||||
assertEquals(repo.asInt(), key3.getRepositoryId());
|
assertEquals(repo.asInt(), key3.getRepositoryId());
|
||||||
assertEquals(key3, id);
|
assertEquals(key3, id);
|
||||||
assertEquals("3e.41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
assertEquals("41234567.3e64b928d51b3a28e89cfe2a3f0eeae35ef07839",
|
||||||
key3.asString());
|
key3.asString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -54,7 +54,7 @@
|
||||||
|
|
||||||
/** Unique identifier of a {@link PackChunk} in the DHT. */
|
/** Unique identifier of a {@link PackChunk} in the DHT. */
|
||||||
public final class ChunkKey implements RowKey {
|
public final class ChunkKey implements RowKey {
|
||||||
static final int KEYLEN = 52;
|
static final int KEYLEN = 49;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param repo
|
* @param repo
|
||||||
|
@ -84,8 +84,8 @@ public static ChunkKey fromBytes(byte[] key, int ptr, int len) {
|
||||||
throw new IllegalArgumentException(MessageFormat.format(
|
throw new IllegalArgumentException(MessageFormat.format(
|
||||||
DhtText.get().invalidChunkKey, decode(key, ptr, ptr + len)));
|
DhtText.get().invalidChunkKey, decode(key, ptr, ptr + len)));
|
||||||
|
|
||||||
int repo = parse32(key, ptr + 3);
|
int repo = parse32(key, ptr);
|
||||||
ObjectId chunk = ObjectId.fromString(key, ptr + 12);
|
ObjectId chunk = ObjectId.fromString(key, ptr + 9);
|
||||||
return new ChunkKey(repo, chunk);
|
return new ChunkKey(repo, chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -122,13 +122,9 @@ public ObjectId getChunkHash() {
|
||||||
|
|
||||||
public byte[] asBytes() {
|
public byte[] asBytes() {
|
||||||
byte[] r = new byte[KEYLEN];
|
byte[] r = new byte[KEYLEN];
|
||||||
chunk.copyTo(r, 12);
|
format32(r, 0, repo);
|
||||||
format32(r, 3, repo);
|
r[8] = '.';
|
||||||
// bucket is the leading 2 digits of the SHA-1.
|
chunk.copyTo(r, 9);
|
||||||
r[11] = '.';
|
|
||||||
r[2] = '.';
|
|
||||||
r[1] = r[12 + 1];
|
|
||||||
r[0] = r[12 + 0];
|
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -55,7 +55,7 @@
|
||||||
|
|
||||||
/** Identifies an ObjectId in the DHT. */
|
/** Identifies an ObjectId in the DHT. */
|
||||||
public final class ObjectIndexKey extends ObjectId implements RowKey {
|
public final class ObjectIndexKey extends ObjectId implements RowKey {
|
||||||
private static final int KEYLEN = 52;
|
private static final int KEYLEN = 49;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param repo
|
* @param repo
|
||||||
|
@ -75,8 +75,8 @@ public static ObjectIndexKey fromBytes(byte[] key) {
|
||||||
throw new IllegalArgumentException(MessageFormat.format(
|
throw new IllegalArgumentException(MessageFormat.format(
|
||||||
DhtText.get().invalidChunkKey, decode(key)));
|
DhtText.get().invalidChunkKey, decode(key)));
|
||||||
|
|
||||||
int repo = parse32(key, 3);
|
int repo = parse32(key, 0);
|
||||||
ObjectId id = ObjectId.fromString(key, 12);
|
ObjectId id = ObjectId.fromString(key, 9);
|
||||||
return new ObjectIndexKey(repo, id);
|
return new ObjectIndexKey(repo, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -106,13 +106,9 @@ int getRepositoryId() {
|
||||||
|
|
||||||
public byte[] asBytes() {
|
public byte[] asBytes() {
|
||||||
byte[] r = new byte[KEYLEN];
|
byte[] r = new byte[KEYLEN];
|
||||||
copyTo(r, 12);
|
format32(r, 0, repo);
|
||||||
format32(r, 3, repo);
|
r[8] = '.';
|
||||||
// bucket is the leading 2 digits of the SHA-1.
|
copyTo(r, 9);
|
||||||
r[11] = '.';
|
|
||||||
r[2] = '.';
|
|
||||||
r[1] = r[12 + 1];
|
|
||||||
r[0] = r[12 + 0];
|
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue