PackWriter: Validate reused cached packs
If object reuse validation is enabled, the output pack is going to probably be stored locally. When reusing an existing cached pack to save object enumeration costs, ensure the cached pack has not been corrupted by checking its SHA-1 trailer. If it has, writing will abort and the output pack won't be complete. This prevents anyone from trying to use the output pack, and catches corruption before it can be carried any further. Change-Id: If89d0d4e429d9f4c86f14de6c0020902705153e6 Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
This commit is contained in:
parent
1b2062fe37
commit
a468cb57c2
|
@ -46,6 +46,7 @@
|
||||||
package org.eclipse.jgit.storage.file;
|
package org.eclipse.jgit.storage.file;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.security.MessageDigest;
|
||||||
import java.util.zip.CRC32;
|
import java.util.zip.CRC32;
|
||||||
import java.util.zip.DataFormatException;
|
import java.util.zip.DataFormatException;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.Inflater;
|
||||||
|
@ -83,8 +84,12 @@ void crc32(CRC32 out, long pos, int cnt) {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void write(PackOutputStream out, long pos, int cnt) throws IOException {
|
void write(PackOutputStream out, long pos, int cnt, MessageDigest digest)
|
||||||
out.write(array, (int) (pos - start), cnt);
|
throws IOException {
|
||||||
|
int ptr = (int) (pos - start);
|
||||||
|
out.write(array, ptr, cnt);
|
||||||
|
if (digest != null)
|
||||||
|
digest.update(array, ptr, cnt);
|
||||||
}
|
}
|
||||||
|
|
||||||
void check(Inflater inf, byte[] tmp, long pos, int cnt)
|
void check(Inflater inf, byte[] tmp, long pos, int cnt)
|
||||||
|
|
|
@ -47,6 +47,7 @@
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
import java.security.MessageDigest;
|
||||||
import java.util.zip.DataFormatException;
|
import java.util.zip.DataFormatException;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.Inflater;
|
||||||
|
|
||||||
|
@ -75,7 +76,8 @@ protected int copy(final int p, final byte[] b, final int o, int n) {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
void write(PackOutputStream out, long pos, int cnt) throws IOException {
|
void write(PackOutputStream out, long pos, int cnt, MessageDigest digest)
|
||||||
|
throws IOException {
|
||||||
final ByteBuffer s = buffer.slice();
|
final ByteBuffer s = buffer.slice();
|
||||||
s.position((int) (pos - start));
|
s.position((int) (pos - start));
|
||||||
|
|
||||||
|
@ -84,6 +86,8 @@ void write(PackOutputStream out, long pos, int cnt) throws IOException {
|
||||||
int n = Math.min(cnt, buf.length);
|
int n = Math.min(cnt, buf.length);
|
||||||
s.get(buf, 0, n);
|
s.get(buf, 0, n);
|
||||||
out.write(buf, 0, n);
|
out.write(buf, 0, n);
|
||||||
|
if (digest != null)
|
||||||
|
digest.update(buf, 0, n);
|
||||||
cnt -= n;
|
cnt -= n;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,6 +45,7 @@
|
||||||
package org.eclipse.jgit.storage.file;
|
package org.eclipse.jgit.storage.file;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.security.MessageDigest;
|
||||||
import java.util.zip.DataFormatException;
|
import java.util.zip.DataFormatException;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.Inflater;
|
||||||
|
|
||||||
|
@ -120,8 +121,8 @@ final int copy(long pos, byte[] dstbuf, int dstoff, int cnt) {
|
||||||
*/
|
*/
|
||||||
protected abstract int copy(int pos, byte[] dstbuf, int dstoff, int cnt);
|
protected abstract int copy(int pos, byte[] dstbuf, int dstoff, int cnt);
|
||||||
|
|
||||||
abstract void write(PackOutputStream out, long pos, int cnt)
|
abstract void write(PackOutputStream out, long pos, int cnt,
|
||||||
throws IOException;
|
MessageDigest md) throws IOException;
|
||||||
|
|
||||||
final int setInput(long pos, Inflater inf) throws DataFormatException {
|
final int setInput(long pos, Inflater inf) throws DataFormatException {
|
||||||
return setInput((int) (pos - start), inf);
|
return setInput((int) (pos - start), inf);
|
||||||
|
|
|
@ -87,9 +87,10 @@ public long getObjectCount() throws IOException {
|
||||||
return cnt;
|
return cnt;
|
||||||
}
|
}
|
||||||
|
|
||||||
void copyAsIs(PackOutputStream out, WindowCursor wc) throws IOException {
|
void copyAsIs(PackOutputStream out, boolean validate, WindowCursor wc)
|
||||||
|
throws IOException {
|
||||||
for (String packName : packNames)
|
for (String packName : packNames)
|
||||||
getPackFile(packName).copyPackAsIs(out, wc);
|
getPackFile(packName).copyPackAsIs(out, validate, wc);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -305,11 +305,11 @@ private final byte[] decompress(final long position, final int sz,
|
||||||
return dstbuf;
|
return dstbuf;
|
||||||
}
|
}
|
||||||
|
|
||||||
void copyPackAsIs(PackOutputStream out, WindowCursor curs)
|
void copyPackAsIs(PackOutputStream out, boolean validate, WindowCursor curs)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// Pin the first window, this ensures the length is accurate.
|
// Pin the first window, this ensures the length is accurate.
|
||||||
curs.pin(this, 0);
|
curs.pin(this, 0);
|
||||||
curs.copyPackAsIs(this, out, 12, length - (12 + 20));
|
curs.copyPackAsIs(this, length, validate, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
|
final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
|
||||||
|
@ -462,7 +462,7 @@ private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
|
||||||
// and we have it pinned. Write this out without copying.
|
// and we have it pinned. Write this out without copying.
|
||||||
//
|
//
|
||||||
out.writeHeader(src, inflatedLength);
|
out.writeHeader(src, inflatedLength);
|
||||||
quickCopy.write(out, dataOffset, (int) dataLength);
|
quickCopy.write(out, dataOffset, (int) dataLength, null);
|
||||||
|
|
||||||
} else if (dataLength <= buf.length) {
|
} else if (dataLength <= buf.length) {
|
||||||
// Tiny optimization: Lots of objects are very small deltas or
|
// Tiny optimization: Lots of objects are very small deltas or
|
||||||
|
@ -499,6 +499,10 @@ boolean invalid() {
|
||||||
return invalid;
|
return invalid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void setInvalid() {
|
||||||
|
invalid = true;
|
||||||
|
}
|
||||||
|
|
||||||
private void readFully(final long position, final byte[] dstbuf,
|
private void readFully(final long position, final byte[] dstbuf,
|
||||||
int dstoff, final int cnt, final WindowCursor curs)
|
int dstoff, final int cnt, final WindowCursor curs)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
|
@ -45,6 +45,9 @@
|
||||||
package org.eclipse.jgit.storage.file;
|
package org.eclipse.jgit.storage.file;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.text.MessageFormat;
|
||||||
|
import java.util.Arrays;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
@ -52,6 +55,7 @@
|
||||||
import java.util.zip.DataFormatException;
|
import java.util.zip.DataFormatException;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.Inflater;
|
||||||
|
|
||||||
|
import org.eclipse.jgit.JGitText;
|
||||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||||
import org.eclipse.jgit.errors.MissingObjectException;
|
import org.eclipse.jgit.errors.MissingObjectException;
|
||||||
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
|
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
|
||||||
|
@ -197,21 +201,52 @@ int copy(final PackFile pack, long position, final byte[] dstbuf,
|
||||||
return cnt - need;
|
return cnt - need;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void copyPackAsIs(PackOutputStream out, CachedPack pack)
|
public void copyPackAsIs(PackOutputStream out, CachedPack pack,
|
||||||
throws IOException {
|
boolean validate) throws IOException {
|
||||||
((LocalCachedPack) pack).copyAsIs(out, this);
|
((LocalCachedPack) pack).copyAsIs(out, validate, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
void copyPackAsIs(final PackFile pack, final PackOutputStream out,
|
void copyPackAsIs(final PackFile pack, final long length, boolean validate,
|
||||||
long position, long cnt) throws IOException {
|
final PackOutputStream out) throws IOException {
|
||||||
while (0 < cnt) {
|
MessageDigest md = null;
|
||||||
|
if (validate) {
|
||||||
|
md = Constants.newMessageDigest();
|
||||||
|
byte[] buf = out.getCopyBuffer();
|
||||||
|
pin(pack, 0);
|
||||||
|
if (window.copy(0, buf, 0, 12) != 12) {
|
||||||
|
pack.setInvalid();
|
||||||
|
throw new IOException(JGitText.get().packfileIsTruncated);
|
||||||
|
}
|
||||||
|
md.update(buf, 0, 12);
|
||||||
|
}
|
||||||
|
|
||||||
|
long position = 12;
|
||||||
|
long remaining = length - (12 + 20);
|
||||||
|
while (0 < remaining) {
|
||||||
pin(pack, position);
|
pin(pack, position);
|
||||||
|
|
||||||
int ptr = (int) (position - window.start);
|
int ptr = (int) (position - window.start);
|
||||||
int n = (int) Math.min(window.size() - ptr, cnt);
|
int n = (int) Math.min(window.size() - ptr, remaining);
|
||||||
window.write(out, position, n);
|
window.write(out, position, n, md);
|
||||||
position += n;
|
position += n;
|
||||||
cnt -= n;
|
remaining -= n;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (md != null) {
|
||||||
|
byte[] buf = new byte[20];
|
||||||
|
byte[] actHash = md.digest();
|
||||||
|
|
||||||
|
pin(pack, position);
|
||||||
|
if (window.copy(position, buf, 0, 20) != 20) {
|
||||||
|
pack.setInvalid();
|
||||||
|
throw new IOException(JGitText.get().packfileIsTruncated);
|
||||||
|
}
|
||||||
|
if (!Arrays.equals(actHash, buf)) {
|
||||||
|
pack.setInvalid();
|
||||||
|
throw new IOException(MessageFormat.format(
|
||||||
|
JGitText.get().packfileCorruptionDetected, pack
|
||||||
|
.getPackFile().getPath()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -216,9 +216,14 @@ public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp,
|
||||||
* stream to append the pack onto.
|
* stream to append the pack onto.
|
||||||
* @param pack
|
* @param pack
|
||||||
* the cached pack to send.
|
* the cached pack to send.
|
||||||
|
* @param validate
|
||||||
|
* if true the representation must be validated and not be
|
||||||
|
* corrupt before being reused. If false, validation may be
|
||||||
|
* skipped as it will be performed elsewhere in the processing
|
||||||
|
* pipeline.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* the pack cannot be read, or stream did not accept a write.
|
* the pack cannot be read, or stream did not accept a write.
|
||||||
*/
|
*/
|
||||||
public abstract void copyPackAsIs(PackOutputStream out, CachedPack pack)
|
public abstract void copyPackAsIs(PackOutputStream out, CachedPack pack,
|
||||||
throws IOException;
|
boolean validate) throws IOException;
|
||||||
}
|
}
|
||||||
|
|
|
@ -660,7 +660,7 @@ public void writePack(ProgressMonitor compressMonitor,
|
||||||
stats.reusedObjects += pack.getObjectCount();
|
stats.reusedObjects += pack.getObjectCount();
|
||||||
stats.reusedDeltas += deltaCnt;
|
stats.reusedDeltas += deltaCnt;
|
||||||
stats.totalDeltas += deltaCnt;
|
stats.totalDeltas += deltaCnt;
|
||||||
reuseSupport.copyPackAsIs(out, pack);
|
reuseSupport.copyPackAsIs(out, pack, reuseValidate);
|
||||||
}
|
}
|
||||||
writeChecksum(out);
|
writeChecksum(out);
|
||||||
out.flush();
|
out.flush();
|
||||||
|
|
Loading…
Reference in New Issue