Merge "ReceivePackStats: Add size and count of unnecessary pushed objects"
This commit is contained in:
commit
292919b12a
|
@ -1059,6 +1059,67 @@ public void testV2FetchIncludeTag() throws Exception {
|
||||||
assertTrue(client.getObjectDatabase().has(tag.toObjectId()));
|
assertTrue(client.getObjectDatabase().has(tag.toObjectId()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUploadNewBytes() throws Exception {
|
||||||
|
String commonInBlob = "abcdefghijklmnopqrstuvwx";
|
||||||
|
|
||||||
|
RevBlob parentBlob = remote.blob(commonInBlob + "a");
|
||||||
|
RevCommit parent = remote.commit(remote.tree(remote.file("foo", parentBlob)));
|
||||||
|
RevBlob childBlob = remote.blob(commonInBlob + "b");
|
||||||
|
RevCommit child = remote.commit(remote.tree(remote.file("foo", childBlob)), parent);
|
||||||
|
remote.update("branch1", child);
|
||||||
|
|
||||||
|
ByteArrayInputStream recvStream = uploadPackV2(
|
||||||
|
"command=fetch\n",
|
||||||
|
PacketLineIn.delimiter(),
|
||||||
|
"want " + child.toObjectId().getName() + "\n",
|
||||||
|
"ofs-delta\n",
|
||||||
|
"done\n",
|
||||||
|
PacketLineIn.end());
|
||||||
|
PacketLineIn pckIn = new PacketLineIn(recvStream);
|
||||||
|
assertThat(pckIn.readString(), is("packfile"));
|
||||||
|
ReceivedPackStatistics receivedStats = parsePack(recvStream);
|
||||||
|
assertTrue(receivedStats.getNumBytesDuplicated() == 0);
|
||||||
|
assertTrue(receivedStats.getNumObjectsDuplicated() == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUploadRedundantBytes() throws Exception {
|
||||||
|
String commonInBlob = "abcdefghijklmnopqrstuvwxyz";
|
||||||
|
|
||||||
|
RevBlob parentBlob = remote.blob(commonInBlob + "a");
|
||||||
|
RevCommit parent = remote.commit(remote.tree(remote.file("foo", parentBlob)));
|
||||||
|
RevBlob childBlob = remote.blob(commonInBlob + "b");
|
||||||
|
RevCommit child = remote.commit(remote.tree(remote.file("foo", childBlob)), parent);
|
||||||
|
remote.update("branch1", child);
|
||||||
|
|
||||||
|
TestRepository<InMemoryRepository> local = new TestRepository<>(client);
|
||||||
|
RevBlob localParentBlob = local.blob(commonInBlob + "a");
|
||||||
|
RevCommit localParent = local.commit(local.tree(local.file("foo", localParentBlob)));
|
||||||
|
RevBlob localChildBlob = local.blob(commonInBlob + "b");
|
||||||
|
RevCommit localChild = local.commit(
|
||||||
|
local.tree(local.file("foo", localChildBlob)), localParent);
|
||||||
|
local.update("branch1", localChild);
|
||||||
|
|
||||||
|
ByteArrayInputStream recvStream = uploadPackV2(
|
||||||
|
"command=fetch\n",
|
||||||
|
PacketLineIn.delimiter(),
|
||||||
|
"want " + child.toObjectId().getName() + "\n",
|
||||||
|
"ofs-delta\n",
|
||||||
|
"done\n",
|
||||||
|
PacketLineIn.end());
|
||||||
|
PacketLineIn pckIn = new PacketLineIn(recvStream);
|
||||||
|
assertThat(pckIn.readString(), is("packfile"));
|
||||||
|
ReceivedPackStatistics receivedStats = parsePack(recvStream);
|
||||||
|
|
||||||
|
long sizeOfHeader = 12;
|
||||||
|
long sizeOfTrailer = 20;
|
||||||
|
long expectedSize = receivedStats.getNumBytesRead() - sizeOfHeader
|
||||||
|
- sizeOfTrailer;
|
||||||
|
assertTrue(receivedStats.getNumBytesDuplicated() == expectedSize);
|
||||||
|
assertTrue(receivedStats.getNumObjectsDuplicated() == 6);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testV2FetchOfsDelta() throws Exception {
|
public void testV2FetchOfsDelta() throws Exception {
|
||||||
String commonInBlob = "abcdefghijklmnopqrstuvwxyz";
|
String commonInBlob = "abcdefghijklmnopqrstuvwxyz";
|
||||||
|
|
|
@ -679,7 +679,8 @@ private void resolveDeltas(DeltaVisit visit, final int type,
|
||||||
|
|
||||||
verifySafeObject(tempObjectId, type, visit.data);
|
verifySafeObject(tempObjectId, type, visit.data);
|
||||||
if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
|
if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
|
||||||
checkObjectCollision(tempObjectId, type, visit.data);
|
checkObjectCollision(tempObjectId, type, visit.data,
|
||||||
|
visit.delta.sizeBeforeInflating);
|
||||||
}
|
}
|
||||||
|
|
||||||
PackedObjectInfo oe;
|
PackedObjectInfo oe;
|
||||||
|
@ -999,6 +1000,7 @@ private void indexOneObject() throws IOException {
|
||||||
UnresolvedDelta n = onEndDelta();
|
UnresolvedDelta n = onEndDelta();
|
||||||
n.position = streamPosition;
|
n.position = streamPosition;
|
||||||
n.next = baseByPos.put(base, n);
|
n.next = baseByPos.put(base, n);
|
||||||
|
n.sizeBeforeInflating = streamPosition() - streamPosition;
|
||||||
deltaCount++;
|
deltaCount++;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -1020,6 +1022,7 @@ private void indexOneObject() throws IOException {
|
||||||
inflateAndSkip(Source.INPUT, sz);
|
inflateAndSkip(Source.INPUT, sz);
|
||||||
UnresolvedDelta n = onEndDelta();
|
UnresolvedDelta n = onEndDelta();
|
||||||
n.position = streamPosition;
|
n.position = streamPosition;
|
||||||
|
n.sizeBeforeInflating = streamPosition() - streamPosition;
|
||||||
r.add(n);
|
r.add(n);
|
||||||
deltaCount++;
|
deltaCount++;
|
||||||
break;
|
break;
|
||||||
|
@ -1071,9 +1074,11 @@ private void whole(long pos, int type, long sz)
|
||||||
verifySafeObject(tempObjectId, type, data);
|
verifySafeObject(tempObjectId, type, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
long sizeBeforeInflating = streamPosition() - pos;
|
||||||
PackedObjectInfo obj = newInfo(tempObjectId, null, null);
|
PackedObjectInfo obj = newInfo(tempObjectId, null, null);
|
||||||
obj.setOffset(pos);
|
obj.setOffset(pos);
|
||||||
obj.setType(type);
|
obj.setType(type);
|
||||||
|
obj.setSize(sizeBeforeInflating);
|
||||||
onEndWholeObject(obj);
|
onEndWholeObject(obj);
|
||||||
if (data != null)
|
if (data != null)
|
||||||
onInflatedObjectData(obj, type, data);
|
onInflatedObjectData(obj, type, data);
|
||||||
|
@ -1148,6 +1153,8 @@ private void checkObjectCollision(PackedObjectInfo obj)
|
||||||
sz -= n;
|
sz -= n;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
stats.incrementObjectsDuplicated();
|
||||||
|
stats.incrementNumBytesDuplicated(obj.getSize());
|
||||||
} catch (MissingObjectException notLocal) {
|
} catch (MissingObjectException notLocal) {
|
||||||
// This is OK, we don't have a copy of the object locally
|
// This is OK, we don't have a copy of the object locally
|
||||||
// but the API throws when we try to read it as usually it's
|
// but the API throws when we try to read it as usually it's
|
||||||
|
@ -1155,15 +1162,17 @@ private void checkObjectCollision(PackedObjectInfo obj)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkObjectCollision(AnyObjectId obj, int type, byte[] data)
|
private void checkObjectCollision(AnyObjectId obj, int type, byte[] data,
|
||||||
throws IOException {
|
long sizeBeforeInflating) throws IOException {
|
||||||
try {
|
try {
|
||||||
final ObjectLoader ldr = readCurs.open(obj, type);
|
final ObjectLoader ldr = readCurs.open(obj, type);
|
||||||
final byte[] existingData = ldr.getCachedBytes(data.length);
|
final byte[] existingData = ldr.getCachedBytes(data.length);
|
||||||
if (!Arrays.equals(data, existingData)) {
|
if (!Arrays.equals(data, existingData)) {
|
||||||
throw new IOException(MessageFormat.format(
|
throw new IOException(MessageFormat
|
||||||
JGitText.get().collisionOn, obj.name()));
|
.format(JGitText.get().collisionOn, obj.name()));
|
||||||
}
|
}
|
||||||
|
stats.incrementObjectsDuplicated();
|
||||||
|
stats.incrementNumBytesDuplicated(sizeBeforeInflating);
|
||||||
} catch (MissingObjectException notLocal) {
|
} catch (MissingObjectException notLocal) {
|
||||||
// This is OK, we don't have a copy of the object locally
|
// This is OK, we don't have a copy of the object locally
|
||||||
// but the API throws when we try to read it as usually its
|
// but the API throws when we try to read it as usually its
|
||||||
|
@ -1653,6 +1662,8 @@ public static class UnresolvedDelta {
|
||||||
|
|
||||||
UnresolvedDelta next;
|
UnresolvedDelta next;
|
||||||
|
|
||||||
|
long sizeBeforeInflating;
|
||||||
|
|
||||||
/** @return offset within the input stream. */
|
/** @return offset within the input stream. */
|
||||||
public long getOffset() {
|
public long getOffset() {
|
||||||
return position;
|
return position;
|
||||||
|
|
|
@ -29,6 +29,8 @@ public class PackedObjectInfo extends ObjectIdOwnerMap.Entry {
|
||||||
|
|
||||||
private int type = Constants.OBJ_BAD;
|
private int type = Constants.OBJ_BAD;
|
||||||
|
|
||||||
|
private long sizeBeforeInflating;
|
||||||
|
|
||||||
PackedObjectInfo(final long headerOffset, final int packedCRC,
|
PackedObjectInfo(final long headerOffset, final int packedCRC,
|
||||||
final AnyObjectId id) {
|
final AnyObjectId id) {
|
||||||
super(id);
|
super(id);
|
||||||
|
@ -108,4 +110,12 @@ public int getType() {
|
||||||
public void setType(int type) {
|
public void setType(int type) {
|
||||||
this.type = type;
|
this.type = type;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void setSize(long sizeBeforeInflating) {
|
||||||
|
this.sizeBeforeInflating = sizeBeforeInflating;
|
||||||
|
}
|
||||||
|
|
||||||
|
long getSize() {
|
||||||
|
return sizeBeforeInflating;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
*/
|
*/
|
||||||
public class ReceivedPackStatistics {
|
public class ReceivedPackStatistics {
|
||||||
private long numBytesRead;
|
private long numBytesRead;
|
||||||
|
private long numBytesDuplicated;
|
||||||
|
|
||||||
private long numWholeCommit;
|
private long numWholeCommit;
|
||||||
private long numWholeTree;
|
private long numWholeTree;
|
||||||
|
@ -26,6 +27,7 @@ public class ReceivedPackStatistics {
|
||||||
private long numWholeTag;
|
private long numWholeTag;
|
||||||
private long numOfsDelta;
|
private long numOfsDelta;
|
||||||
private long numRefDelta;
|
private long numRefDelta;
|
||||||
|
private long numObjectsDuplicated;
|
||||||
|
|
||||||
private long numDeltaCommit;
|
private long numDeltaCommit;
|
||||||
private long numDeltaTree;
|
private long numDeltaTree;
|
||||||
|
@ -41,6 +43,17 @@ public long getNumBytesRead() {
|
||||||
return numBytesRead;
|
return numBytesRead;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get number of bytes of objects already in the local database
|
||||||
|
*
|
||||||
|
* @return number of bytes of objects appeared in both the pack sent by the
|
||||||
|
* client and the local database
|
||||||
|
* @since 5.10
|
||||||
|
*/
|
||||||
|
public long getNumBytesDuplicated() {
|
||||||
|
return numBytesDuplicated;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get number of whole commit objects in the pack
|
* Get number of whole commit objects in the pack
|
||||||
*
|
*
|
||||||
|
@ -95,6 +108,17 @@ public long getNumRefDelta() {
|
||||||
return numRefDelta;
|
return numRefDelta;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get number of objects already in the local database
|
||||||
|
*
|
||||||
|
* @return number of objects appeared in both the pack sent by the client
|
||||||
|
* and the local database
|
||||||
|
* @since 5.10
|
||||||
|
*/
|
||||||
|
public long getNumObjectsDuplicated() {
|
||||||
|
return numObjectsDuplicated;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get number of delta commit objects in the pack
|
* Get number of delta commit objects in the pack
|
||||||
*
|
*
|
||||||
|
@ -134,6 +158,7 @@ public long getNumDeltaTag() {
|
||||||
/** A builder for {@link ReceivedPackStatistics}. */
|
/** A builder for {@link ReceivedPackStatistics}. */
|
||||||
public static class Builder {
|
public static class Builder {
|
||||||
private long numBytesRead;
|
private long numBytesRead;
|
||||||
|
private long numBytesDuplicated;
|
||||||
|
|
||||||
private long numWholeCommit;
|
private long numWholeCommit;
|
||||||
private long numWholeTree;
|
private long numWholeTree;
|
||||||
|
@ -141,6 +166,7 @@ public static class Builder {
|
||||||
private long numWholeTag;
|
private long numWholeTag;
|
||||||
private long numOfsDelta;
|
private long numOfsDelta;
|
||||||
private long numRefDelta;
|
private long numRefDelta;
|
||||||
|
private long numObjectsDuplicated;
|
||||||
|
|
||||||
private long numDeltaCommit;
|
private long numDeltaCommit;
|
||||||
private long numDeltaTree;
|
private long numDeltaTree;
|
||||||
|
@ -156,6 +182,17 @@ public Builder setNumBytesRead(long numBytesRead) {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param size
|
||||||
|
* additional bytes already in the local database
|
||||||
|
* @return this
|
||||||
|
* @since 5.10
|
||||||
|
*/
|
||||||
|
Builder incrementNumBytesDuplicated(long size) {
|
||||||
|
numBytesDuplicated += size;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment a whole object count.
|
* Increment a whole object count.
|
||||||
*
|
*
|
||||||
|
@ -195,6 +232,17 @@ public Builder addRefDelta() {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increment the duplicated object count.
|
||||||
|
*
|
||||||
|
* @return this
|
||||||
|
* @since 5.10
|
||||||
|
*/
|
||||||
|
Builder incrementObjectsDuplicated() {
|
||||||
|
numObjectsDuplicated++;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment a delta object count.
|
* Increment a delta object count.
|
||||||
*
|
*
|
||||||
|
@ -226,6 +274,7 @@ public Builder addDeltaObject(int type) {
|
||||||
ReceivedPackStatistics build() {
|
ReceivedPackStatistics build() {
|
||||||
ReceivedPackStatistics s = new ReceivedPackStatistics();
|
ReceivedPackStatistics s = new ReceivedPackStatistics();
|
||||||
s.numBytesRead = numBytesRead;
|
s.numBytesRead = numBytesRead;
|
||||||
|
s.numBytesDuplicated = numBytesDuplicated;
|
||||||
s.numWholeCommit = numWholeCommit;
|
s.numWholeCommit = numWholeCommit;
|
||||||
s.numWholeTree = numWholeTree;
|
s.numWholeTree = numWholeTree;
|
||||||
s.numWholeBlob = numWholeBlob;
|
s.numWholeBlob = numWholeBlob;
|
||||||
|
@ -236,6 +285,7 @@ ReceivedPackStatistics build() {
|
||||||
s.numDeltaTree = numDeltaTree;
|
s.numDeltaTree = numDeltaTree;
|
||||||
s.numDeltaBlob = numDeltaBlob;
|
s.numDeltaBlob = numDeltaBlob;
|
||||||
s.numDeltaTag = numDeltaTag;
|
s.numDeltaTag = numDeltaTag;
|
||||||
|
s.numObjectsDuplicated = numObjectsDuplicated;
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue