Allow object reuse selection to occur in parallel

ObjectReader implementations may wish to use multiple threads in
order to evaluate object reuse faster.  Let the reader make that
decision by passing the iteration down into the reader.

Because the work is pushed into the reader, it may need to locate a
given ObjectToPack given its ObjectId.  This can easily occur if the
reader has sent a list of ObjectIds to the object database and gets
back information keyed only by ObjectId, without the ObjectToPack
handle.  Expose lookup using the PackWriter's own internal map,
so the reader doesn't need to build a redundant copy to track the
assocation of ObjectId back to ObjectToPack.

Change-Id: I0c536405a55034881fb5db92a2d2a99534faed34
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java

@@ -46,6 +46,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.zip.InflaterInputStream;
 
 import org.eclipse.jgit.errors.MissingObjectException;
@@ -103,7 +104,8 @@ public void select(ObjectToPack otp, StoredObjectRepresentation next) {
 		};
 
 		ByteArrayOutputStream buf = new ByteArrayOutputStream();
-		asis.selectObjectRepresentation(pw, target);
+		asis.selectObjectRepresentation(pw, NullProgressMonitor.INSTANCE,
+				Collections.singleton(target));
 		asis.copyObjectAsIs(new PackOutputStream(NullProgressMonitor.INSTANCE,
 				buf, pw), target);
 

org.eclipse.jgit/resources/org/eclipse/jgit/JGitText.properties

@@ -321,6 +321,7 @@ repositoryState_rebaseOrApplyMailbox=Rebase/Apply mailbox
 repositoryState_rebaseWithMerge=Rebase w/merge
 requiredHashFunctionNotAvailable=Required hash function {0} not available.
 resolvingDeltas=Resolving deltas
+searchForReuse=Finding sources
 serviceNotPermitted={0} not permitted
 shortCompressedStreamAt=Short compressed stream at {0}
 shortReadOfBlock=Short read of block.

org.eclipse.jgit/src/org/eclipse/jgit/JGitText.java

@@ -380,6 +380,7 @@ public static JGitText get() {
 	/***/ public String repositoryState_rebaseWithMerge;
 	/***/ public String requiredHashFunctionNotAvailable;
 	/***/ public String resolvingDeltas;
+	/***/ public String searchForReuse;
 	/***/ public String serviceNotPermitted;
 	/***/ public String shortCompressedStreamAt;
 	/***/ public String shortReadOfBlock;

org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCursor.java

@@ -56,6 +56,7 @@
 import org.eclipse.jgit.lib.InflaterCache;
 import org.eclipse.jgit.lib.ObjectLoader;
 import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.revwalk.RevObject;
 import org.eclipse.jgit.storage.pack.ObjectReuseAsIs;
 import org.eclipse.jgit.storage.pack.ObjectToPack;
@@ -116,9 +117,13 @@ public LocalObjectToPack newObjectToPack(RevObject obj) {
 		return new LocalObjectToPack(obj);
 	}
 
-	public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp)
+	public void selectObjectRepresentation(PackWriter packer,
+			ProgressMonitor monitor, Iterable<ObjectToPack> objects)
 			throws IOException, MissingObjectException {
-		db.selectObjectRepresentation(packer, otp, this);
+		for (ObjectToPack otp : objects) {
+			db.selectObjectRepresentation(packer, otp, this);
+			monitor.update(1);
+		}
 	}
 
 	public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp)

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ObjectReuseAsIs.java

@@ -48,6 +48,7 @@
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
 import org.eclipse.jgit.lib.ObjectReader;
+import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.revwalk.RevObject;
 
 /**
@@ -78,24 +79,32 @@ public interface ObjectReuseAsIs {
 
 	/**
 	 * Select the best object representation for a packer.
-	 * <p>
+	 *
 	 * Implementations should iterate through all available representations of
 	 * an object, and pass them in turn to the PackWriter though
 	 * {@link PackWriter#select(ObjectToPack, StoredObjectRepresentation)} so
 	 * the writer can select the most suitable representation to reuse into the
 	 * output stream.
 	 *
+	 * The implementation may choose to consider multiple objects at once on
+	 * concurrent threads, but must evaluate all representations of an object
+	 * within the same thread.
+	 *
 	 * @param packer
 	 *            the packer that will write the object in the near future.
-	 * @param otp
-	 *            the object to pack.
+	 * @param monitor
+	 *            progress monitor, implementation should update the monitor
+	 *            once for each item in the iteration when selection is done.
+	 * @param objects
+	 *            the objects that are being packed.
 	 * @throws MissingObjectException
 	 *             there is no representation available for the object, as it is
 	 *             no longer in the repository. Packing will abort.
 	 * @throws IOException
 	 *             the repository cannot be accessed. Packing will abort.
 	 */
-	public void selectObjectRepresentation(PackWriter packer, ObjectToPack otp)
+	public void selectObjectRepresentation(PackWriter packer,
+			ProgressMonitor monitor, Iterable<ObjectToPack> objects)
 			throws IOException, MissingObjectException;
 
 	/**

org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java

@@ -388,7 +388,18 @@ public void preparePack(ProgressMonitor countingMonitor,
 	 * @return true if the object will appear in the output pack file.
 	 */
 	public boolean willInclude(final AnyObjectId id) {
-		return objectsMap.get(id) != null;
+		return get(id) != null;
+	}
+
+	/**
+	 * Lookup the ObjectToPack object for a given ObjectId.
+	 *
+	 * @param id
+	 *            the object to find in the pack.
+	 * @return the object we are packing, or null.
+	 */
+	public ObjectToPack get(AnyObjectId id) {
+		return objectsMap.get(id);
 	}
 
 	/**
@@ -477,7 +488,7 @@ public void writePack(ProgressMonitor compressMonitor,
 			writeMonitor = NullProgressMonitor.INSTANCE;
 
 		if ((reuseDeltas || config.isReuseObjects()) && reuseSupport != null)
-			searchForReuse();
+			searchForReuse(compressMonitor);
 		if (config.isDeltaCompress())
 			searchForDeltas(compressMonitor);
 
@@ -504,11 +515,11 @@ public void release() {
 		}
 	}
 
-	private void searchForReuse() throws IOException {
-		for (List<ObjectToPack> list : objectsLists) {
-			for (ObjectToPack otp : list)
-				reuseSupport.selectObjectRepresentation(this, otp);
-		}
+	private void searchForReuse(ProgressMonitor monitor) throws IOException {
+		monitor.beginTask(JGitText.get().searchForReuse, getObjectsNumber());
+		for (List<ObjectToPack> list : objectsLists)
+			reuseSupport.selectObjectRepresentation(this, monitor, list);
+		monitor.endTask();
 	}
 
 	private void searchForDeltas(ProgressMonitor monitor)
@@ -840,7 +851,8 @@ private void redoSearchForReuse(final ObjectToPack otp) throws IOException,
 			MissingObjectException {
 		otp.clearDeltaBase();
 		otp.clearReuseAsIs();
-		reuseSupport.selectObjectRepresentation(this, otp);
+		reuseSupport.selectObjectRepresentation(this,
+				NullProgressMonitor.INSTANCE, Collections.singleton(otp));
 	}
 
 	private void writeWholeObjectDeflate(PackOutputStream out,