motiejus
/
jgit
1
Fork 0
Code Releases Activity

Switch to pure Java SHA1 for ObjectId 

Generate names for objects using only the pure Java SHA1
implementation, but continue using MessageDigest in tests.
This opens the possibility of changing the hashing function
to incorporate additional safety measures, such as those
used in sha1dc[1].

Since MessageDigest has higher throughput, continue using
MessageDigest for computing pack, idx and DirCache trailers.
These are less likely to be sensitive to SHAttered[2] types
of attacks, as Git uses them to detect random bit flips
during transfer, and not for content identity.

[1] https://github.com/cr-marcstevens/sha1collisiondetection
[2] https://shattered.it/

Change-Id: If6da98334201f7f20cb916e46f782c45f373784e
This commit is contained in:
Shawn Pearce 2017-02-24 23:33:18 -08:00
parent 982f5d1bf1
commit 0f25f64d48
8 changed files with 104 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java
View File

@ -90,6 +90,7 @@
import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.TemporaryBuffer;
import org.eclipse.jgit.util.io.CountingOutputStream;
import org.eclipse.jgit.util.sha1.SHA1;
/** Inserts objects into the DFS. */
public class DfsInserter extends ObjectInserter {
@ -168,7 +169,7 @@ public ObjectId insert(int type, long len, InputStream in)
}
long offset = beginObject(type, len);
MessageDigest md = digest();
SHA1 md = SHA1.newInstance();
md.update(Constants.encodedTypeString(type));
md.update((byte) ' ');
md.update(Constants.encodeASCII(len));
@ -183,7 +184,7 @@ public ObjectId insert(int type, long len, InputStream in)
len -= n;
}
packOut.compress.finish();
return endObject(ObjectId.fromRaw(md.digest()), offset);
return endObject(md.toObjectId(), offset);
}
private byte[] insertBuffer(long len) {

33
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryInserter.java
View File

@ -49,12 +49,11 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
@ -69,6 +68,7 @@
import org.eclipse.jgit.transport.PackParser;
import org.eclipse.jgit.util.FileUtils;
import org.eclipse.jgit.util.IO;
import org.eclipse.jgit.util.sha1.SHA1;
/** Creates loose objects in a {@link ObjectDirectory}. */
class ObjectDirectoryInserter extends ObjectInserter {
@ -140,9 +140,9 @@ ObjectId insert(int type, long len, InputStream is, boolean createDuplicate)
return insert(type, buf, 0, actLen, createDuplicate);
} else {
MessageDigest md = digest();
SHA1 md = SHA1.newInstance();
File tmp = toTemp(md, type, len, is);
ObjectId id = ObjectId.fromRaw(md.digest());
ObjectId id = md.toObjectId();
return insertOneObject(tmp, id, createDuplicate);
}
}
@ -193,7 +193,7 @@ public void close() {
}
@SuppressWarnings("resource" /* java 7 */)
private File toTemp(final MessageDigest md, final int type, long len,
private File toTemp(final SHA1 md, final int type, long len,
final InputStream is) throws IOException, FileNotFoundException,
Error {
boolean delete = true;
@ -205,7 +205,7 @@ private File toTemp(final MessageDigest md, final int type, long len,
if (config.getFSyncObjectFiles())
out = Channels.newOutputStream(fOut.getChannel());
DeflaterOutputStream cOut = compress(out);
DigestOutputStream dOut = new DigestOutputStream(cOut, md);
SHA1OutputStream dOut = new SHA1OutputStream(cOut, md);
writeHeader(dOut, type, len);
final byte[] buf = buffer();
@ -285,4 +285,25 @@ private static EOFException shortInput(long missing) {
return new EOFException(MessageFormat.format(
JGitText.get().inputDidntMatchLength, Long.valueOf(missing)));
}
private static class SHA1OutputStream extends FilterOutputStream {
private final SHA1 md;
SHA1OutputStream(OutputStream out, SHA1 md) {
super(out);
this.md = md;
}
@Override
public void write(int b) throws IOException {
md.update((byte) b);
out.write(b);
}
@Override
public void write(byte[] in, int p, int n) throws IOException {
md.update(in, p, n);
out.write(in, p, n);
}
}
}

18
org.eclipse.jgit/src/org/eclipse/jgit/lib/MutableObjectId.java
View File

@ -208,6 +208,24 @@ public void fromRaw(final int[] ints, final int p) {
w5 = ints[p + 4];
}
/**
* Convert an ObjectId from binary representation expressed in integers.
*
* @param a
* @param b
* @param c
* @param d
* @param e
* @since 4.7
*/
public void set(int a, int b, int c, int d, int e) {
w1 = a;
w2 = b;
w3 = c;
w4 = d;
w5 = e;
}
/**
* Convert an ObjectId from hex characters (US-ASCII).
*

13
org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java
View File

@ -248,8 +248,17 @@ private static final ObjectId fromHexString(final byte[] bs, int p) {
}
}
ObjectId(final int new_1, final int new_2, final int new_3,
final int new_4, final int new_5) {
/**
* Construct an ObjectId from 160 bits provided in 5 words.
*
* @param new_1
* @param new_2
* @param new_3
* @param new_4
* @param new_5
* @since 4.7
*/
public ObjectId(int new_1, int new_2, int new_3, int new_4, int new_5) {
w1 = new_1;
w2 = new_2;
w3 = new_3;

19
org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectInserter.java
View File

@ -50,10 +50,10 @@
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.transport.PackParser;
import org.eclipse.jgit.util.sha1.SHA1;
/**
* Inserts objects into an existing {@code ObjectDatabase}.
@ -177,15 +177,11 @@ public void close() {
}
}
/** Digest to compute the name of an object. */
private final MessageDigest digest;
/** Temporary working buffer for streaming data through. */
private byte[] tempBuffer;
/** Create a new inserter for a database. */
protected ObjectInserter() {
digest = Constants.newMessageDigest();
}
/**
@ -220,9 +216,8 @@ protected byte[] buffer() {
}
/** @return digest to help compute an ObjectId */
protected MessageDigest digest() {
digest.reset();
return digest;
protected SHA1 digest() {
return SHA1.newInstance();
}
/**
@ -252,13 +247,13 @@ public ObjectId idFor(int type, byte[] data) {
* @return the name of the object.
*/
public ObjectId idFor(int type, byte[] data, int off, int len) {
MessageDigest md = digest();
SHA1 md = SHA1.newInstance();
md.update(Constants.encodedTypeString(type));
md.update((byte) ' ');
md.update(Constants.encodeASCII(len));
md.update((byte) 0);
md.update(data, off, len);
return ObjectId.fromRaw(md.digest());
return md.toObjectId();
}
/**
@ -277,7 +272,7 @@ public ObjectId idFor(int type, byte[] data, int off, int len) {
*/
public ObjectId idFor(int objectType, long length, InputStream in)
throws IOException {
MessageDigest md = digest();
SHA1 md = SHA1.newInstance();
md.update(Constants.encodedTypeString(objectType));
md.update((byte) ' ');
md.update(Constants.encodeASCII(length));
@ -290,7 +285,7 @@ public ObjectId idFor(int objectType, long length, InputStream in)
md.update(buf, 0, n);
length -= n;
}
return ObjectId.fromRaw(md.digest());
return md.toObjectId();
}
/**

12
org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java
View File

@ -83,6 +83,7 @@
import org.eclipse.jgit.util.BlockList;
import org.eclipse.jgit.util.IO;
import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.sha1.SHA1;
/**
* Parses a pack stream and imports it for an {@link ObjectInserter}.
@ -116,8 +117,6 @@ public static enum Source {
private byte[] hdrBuf;
private final MessageDigest objectDigest;
private final MutableObjectId tempObjectId;
private InputStream in;
@ -206,7 +205,6 @@ protected PackParser(final ObjectDatabase odb, final InputStream src) {
buf = new byte[BUFFER_SIZE];
tempBuffer = new byte[BUFFER_SIZE];
hdrBuf = new byte[64];
objectDigest = Constants.newMessageDigest();
tempObjectId = new MutableObjectId();
packDigest = Constants.newMessageDigest();
checkObjectCollisions = true;
@ -667,12 +665,13 @@ private void resolveDeltas(DeltaVisit visit, final int type,
JGitText.get().corruptionDetectedReReadingAt,
Long.valueOf(visit.delta.position)));
SHA1 objectDigest = SHA1.newInstance();
objectDigest.update(Constants.encodedTypeString(type));
objectDigest.update((byte) ' ');
objectDigest.update(Constants.encodeASCII(visit.data.length));
objectDigest.update((byte) 0);
objectDigest.update(visit.data);
tempObjectId.fromRaw(objectDigest.digest(), 0);
objectDigest.digest(tempObjectId);
verifySafeObject(tempObjectId, type, visit.data);
@ -1024,6 +1023,7 @@ private void indexOneObject() throws IOException {
private void whole(final long pos, final int type, final long sz)
throws IOException {
SHA1 objectDigest = SHA1.newInstance();
objectDigest.update(Constants.encodedTypeString(type));
objectDigest.update((byte) ' ');
objectDigest.update(Constants.encodeASCII(sz));
@ -1043,7 +1043,7 @@ private void whole(final long pos, final int type, final long sz)
cnt += r;
}
inf.close();
tempObjectId.fromRaw(objectDigest.digest(), 0);
objectDigest.digest(tempObjectId);
checkContentLater = isCheckObjectCollisions()
&& readCurs.has(tempObjectId);
data = null;
@ -1051,7 +1051,7 @@ private void whole(final long pos, final int type, final long sz)
} else {
data = inflateAndReturn(Source.INPUT, sz);
objectDigest.update(data);
tempObjectId.fromRaw(objectDigest.digest(), 0);
objectDigest.digest(tempObjectId);
verifySafeObject(tempObjectId, type, data);
}

15
org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
View File

@ -56,7 +56,6 @@
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetEncoder;
import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.Arrays;
import java.util.Collections;
@ -99,6 +98,7 @@
import org.eclipse.jgit.util.TemporaryBuffer.LocalFile;
import org.eclipse.jgit.util.io.AutoLFInputStream;
import org.eclipse.jgit.util.io.EolStreamTypeUtil;
import org.eclipse.jgit.util.sha1.SHA1;
/**
* Walks a working directory tree as part of a {@link TreeWalk}.
@ -364,7 +364,7 @@ private byte[] idBufferBlob(final Entry e) {
if (is == null)
return zeroid;
try {
state.initializeDigestAndReadBuffer();
state.initializeReadBuffer();
final long len = e.getLength();
InputStream filteredIs = possiblyFilteredInputStream(e, is, len,
@ -1099,10 +1099,9 @@ private static long computeLength(InputStream in) throws IOException {
}
private byte[] computeHash(InputStream in, long length) throws IOException {
final MessageDigest contentDigest = state.contentDigest;
SHA1 contentDigest = SHA1.newInstance();
final byte[] contentReadBuffer = state.contentReadBuffer;
contentDigest.reset();
contentDigest.update(hblob);
contentDigest.update((byte) ' ');
@ -1330,9 +1329,6 @@ private static final class IteratorState {
/** File name character encoder. */
final CharsetEncoder nameEncoder;
/** Digest computer for {@link #contentId} computations. */
MessageDigest contentDigest;
/** Buffer used to perform {@link #contentId} computations. */
byte[] contentReadBuffer;
@ -1347,9 +1343,8 @@ private static final class IteratorState {
this.nameEncoder = Constants.CHARSET.newEncoder();
}
void initializeDigestAndReadBuffer() {
if (contentDigest == null) {
contentDigest = Constants.newMessageDigest();
void initializeReadBuffer() {
if (contentReadBuffer == null) {
contentReadBuffer = new byte[BUFFER_SIZE];
}
}

27
org.eclipse.jgit/src/org/eclipse/jgit/util/sha1/SHA1.java
View File

@ -45,6 +45,8 @@
import java.util.Arrays;
import org.eclipse.jgit.lib.MutableObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.util.NB;
/**
@ -259,4 +261,29 @@ public byte[] digest() {
NB.encodeInt32(b, 16, h4);
return b;
}
/**
* Finish the digest and return the resulting hash.
* <p>
* Once {@code digest()} is called, this instance should be discarded.
*
* @return the ObjectId for the resulting hash.
*/
public ObjectId toObjectId() {
finish();
return new ObjectId(h0, h1, h2, h3, h4);
}
/**
* Finish the digest and return the resulting hash.
* <p>
* Once {@code digest()} is called, this instance should be discarded.
*
* @param id
* destination to copy the digest to.
*/
public void digest(MutableObjectId id) {
finish();
id.set(h0, h1, h2, h3, h4);
}
}