From d9bbb04c3ef28db1d2e492506a819b7ac5c5f9df Mon Sep 17 00:00:00 2001 From: Terry Parker Date: Fri, 12 Jun 2015 12:00:36 -0700 Subject: [PATCH] Introduce PostUploadHook to replace UploadPackLogger UploadPackLogger is incorrectly named--it can be used to trigger any post upload action, such as GC/compaction. This change introduces PostUploadHook/PostUploadHookChain to replace UploadPackLogger/UploadPackLoggerChain and deprecates the latter. It also introduces PackStatistics as a replacement for PackWriter.Statistics, since the latter is not public API. It changes PackWriter to use PackStatistics and reimplements PackWriter.Statistics to delegate to PackStatistics. Change-Id: Ic51df1613e471f568ffee25ae67e118425b38986 Signed-off-by: Terry Parker --- .../storage/dfs/DfsGarbageCollector.java | 9 +- .../storage/dfs/DfsPackCompactor.java | 9 +- .../storage/dfs/DfsPackDescription.java | 8 +- .../internal/storage/pack/PackWriter.java | 163 +++--- .../jgit/storage/pack/PackStatistics.java | 462 ++++++++++++++++++ .../jgit/transport/PostUploadHook.java | 74 +++ .../jgit/transport/PostUploadHookChain.java | 90 ++++ .../eclipse/jgit/transport/UploadPack.java | 57 ++- .../jgit/transport/UploadPackLogger.java | 3 + .../jgit/transport/UploadPackLoggerChain.java | 5 +- 10 files changed, 766 insertions(+), 114 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackStatistics.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHook.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHookChain.java diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index de6629220..faf27e32b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -72,6 +72,7 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.util.io.CountingOutputStream; /** Repack and garbage collect a repository. */ @@ -84,7 +85,7 @@ public class DfsGarbageCollector { private final List newPackDesc; - private final List newPackStats; + private final List newPackStats; private final List newPackObj; @@ -115,7 +116,7 @@ public DfsGarbageCollector(DfsRepository repository) { refdb = repo.getRefDatabase(); objdb = repo.getObjectDatabase(); newPackDesc = new ArrayList(4); - newPackStats = new ArrayList(4); + newPackStats = new ArrayList(4); newPackObj = new ArrayList(4); packConfig = new PackConfig(repo); @@ -258,7 +259,7 @@ public List getNewPacks() { } /** @return statistics corresponding to the {@link #getNewPacks()}. */ - public List getNewPackStatistics() { + public List getNewPackStatistics() { return newPackStats; } @@ -396,7 +397,7 @@ public boolean contains(AnyObjectId objectId) { } }); - PackWriter.Statistics stats = pw.getStatistics(); + PackStatistics stats = pw.getStatistics(); pack.setPackStats(stats); newPackStats.add(stats); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index dbe72b2d4..7073763a7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -67,6 +67,7 @@ import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.io.CountingOutputStream; @@ -94,7 +95,7 @@ public class DfsPackCompactor { private final List newPacks; - private final List newStats; + private final List newStats; private int autoAddSize; @@ -114,7 +115,7 @@ public DfsPackCompactor(DfsRepository repository) { srcPacks = new ArrayList(); exclude = new ArrayList(4); newPacks = new ArrayList(1); - newStats = new ArrayList(1); + newStats = new ArrayList(1); } /** @@ -231,7 +232,7 @@ public void compact(ProgressMonitor pm) throws IOException { writePack(objdb, pack, pw, pm); writeIndex(objdb, pack, pw); - PackWriter.Statistics stats = pw.getStatistics(); + PackStatistics stats = pw.getStatistics(); pw.close(); pw = null; @@ -264,7 +265,7 @@ public List getNewPacks() { } /** @return statistics corresponding to the {@link #getNewPacks()}. */ - public List getNewPackStatistics() { + public List getNewPackStatistics() { return newStats; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java index fba515794..2b9d0e55c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java @@ -50,7 +50,7 @@ import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.storage.pack.PackStatistics; /** * Description of a DFS stored pack/index file. @@ -75,7 +75,7 @@ public class DfsPackDescription implements Comparable { private long deltaCount; - private PackWriter.Statistics stats; + private PackStatistics stats; private int extensions; @@ -225,11 +225,11 @@ public DfsPackDescription setDeltaCount(long cnt) { * DfsGarbageCollector or DfsPackCompactor, and only when the pack * is being committed to the repository. */ - public PackWriter.Statistics getPackStats() { + public PackStatistics getPackStats() { return stats; } - DfsPackDescription setPackStats(PackWriter.Statistics stats) { + DfsPackDescription setPackStats(PackStatistics stats) { this.stats = stats; setFileSize(PACK, stats.getTotalBytes()); setObjectCount(stats.getTotalObjects()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index af02f4ced..43ec19818 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -114,6 +114,7 @@ import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.transport.ObjectCountCallback; import org.eclipse.jgit.transport.WriteAbortedException; import org.eclipse.jgit.util.BlockList; @@ -247,13 +248,13 @@ public static Iterable getInstances() { private final PackConfig config; - private final Statistics stats; + private final PackStatistics.Accumulator stats; private final MutableState state; private final WeakReference selfRef; - private Statistics.ObjectType typeStats; + private PackStatistics.ObjectType.Accumulator typeStats; private List sortedByName; @@ -356,7 +357,7 @@ public PackWriter(final PackConfig config, final ObjectReader reader) { deltaBaseAsOffset = config.isDeltaBaseAsOffset(); reuseDeltas = config.isReuseDeltas(); reuseValidate = true; // be paranoid by default - stats = new Statistics(); + stats = new PackStatistics.Accumulator(); state = new MutableState(); selfRef = new WeakReference(this); instances.put(selfRef, Boolean.TRUE); @@ -981,7 +982,7 @@ public void writePack(ProgressMonitor compressMonitor, writeObjects(out); if (!edgeObjects.isEmpty() || !cachedPacks.isEmpty()) { - for (Statistics.ObjectType typeStat : stats.objectTypes) { + for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) { if (typeStat == null) continue; stats.thinPackBytes += typeStat.bytes; @@ -1002,7 +1003,7 @@ public void writePack(ProgressMonitor compressMonitor, stats.timeWriting = System.currentTimeMillis() - writeStart; stats.depth = depth; - for (Statistics.ObjectType typeStat : stats.objectTypes) { + for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) { if (typeStat == null) continue; typeStat.cntDeltas += typeStat.reusedDeltas; @@ -1022,8 +1023,8 @@ public void writePack(ProgressMonitor compressMonitor, * final pack stream. The object is only available to callers after * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)} */ - public Statistics getStatistics() { - return stats; + public PackStatistics getStatistics() { + return new PackStatistics(stats); } /** @return snapshot of the current state of this PackWriter. */ @@ -2028,28 +2029,36 @@ private boolean reuseDeltaFor(ObjectToPack otp) { return true; } - /** Summary of how PackWriter created the pack. */ + /** + * Summary of how PackWriter created the pack. + * + * @deprecated Use {@link PackStatistics} instead. + */ + @Deprecated public static class Statistics { /** Statistics about a single class of object. */ public static class ObjectType { - long cntObjects; + // All requests are forwarded to this object. + private PackStatistics.ObjectType objectType; - long cntDeltas; - - long reusedObjects; - - long reusedDeltas; - - long bytes; - - long deltaBytes; + /** + * Wraps an + * {@link org.eclipse.jgit.storage.pack.PackStatistics.ObjectType} + * instance to maintain backwards compatibility with existing API. + * + * @param type + * the wrapped instance + */ + public ObjectType(PackStatistics.ObjectType type) { + objectType = type; + } /** * @return total number of objects output. This total includes the * value of {@link #getDeltas()}. */ public long getObjects() { - return cntObjects; + return objectType.getObjects(); } /** @@ -2057,7 +2066,7 @@ public long getObjects() { * actual number of deltas if a cached pack was reused. */ public long getDeltas() { - return cntDeltas; + return objectType.getDeltas(); } /** @@ -2066,7 +2075,7 @@ public long getDeltas() { * {@link #getReusedDeltas()}. */ public long getReusedObjects() { - return reusedObjects; + return objectType.getReusedObjects(); } /** @@ -2077,7 +2086,7 @@ public long getReusedObjects() { * was reused. */ public long getReusedDeltas() { - return reusedDeltas; + return objectType.getReusedDeltas(); } /** @@ -2086,7 +2095,7 @@ public long getReusedDeltas() { * also includes all of {@link #getDeltaBytes()}. */ public long getBytes() { - return bytes; + return objectType.getBytes(); } /** @@ -2094,54 +2103,22 @@ public long getBytes() { * object headers for the delta objects. */ public long getDeltaBytes() { - return deltaBytes; + return objectType.getDeltaBytes(); } } - Set interestingObjects; + // All requests are forwarded to this object. + private PackStatistics statistics; - Set uninterestingObjects; - - Collection reusedPacks; - - int depth; - - int deltaSearchNonEdgeObjects; - - int deltasFound; - - long totalObjects; - - long bitmapIndexMisses; - - long totalDeltas; - - long reusedObjects; - - long reusedDeltas; - - long totalBytes; - - long thinPackBytes; - - long timeCounting; - - long timeSearchingForReuse; - - long timeSearchingForSizes; - - long timeCompressing; - - long timeWriting; - - ObjectType[] objectTypes; - - { - objectTypes = new ObjectType[5]; - objectTypes[OBJ_COMMIT] = new ObjectType(); - objectTypes[OBJ_TREE] = new ObjectType(); - objectTypes[OBJ_BLOB] = new ObjectType(); - objectTypes[OBJ_TAG] = new ObjectType(); + /** + * Wraps a {@link PackStatistics} object to maintain backwards + * compatibility with existing API. + * + * @param stats + * the wrapped PackStatitics object + */ + public Statistics(PackStatistics stats) { + statistics = stats; } /** @@ -2150,7 +2127,7 @@ public long getDeltaBytes() { * test. */ public Set getInterestingObjects() { - return interestingObjects; + return statistics.getInterestingObjects(); } /** @@ -2159,7 +2136,7 @@ public Set getInterestingObjects() { * has these objects. */ public Set getUninterestingObjects() { - return uninterestingObjects; + return statistics.getUninterestingObjects(); } /** @@ -2167,7 +2144,7 @@ public Set getUninterestingObjects() { * in the output, if any were selected for reuse. */ public Collection getReusedPacks() { - return reusedPacks; + return statistics.getReusedPacks(); } /** @@ -2175,7 +2152,7 @@ public Collection getReusedPacks() { * delta search process in order to find a potential delta base. */ public int getDeltaSearchNonEdgeObjects() { - return deltaSearchNonEdgeObjects; + return statistics.getDeltaSearchNonEdgeObjects(); } /** @@ -2184,7 +2161,7 @@ public int getDeltaSearchNonEdgeObjects() { * {@link #getDeltaSearchNonEdgeObjects()}. */ public int getDeltasFound() { - return deltasFound; + return statistics.getDeltasFound(); } /** @@ -2192,7 +2169,7 @@ public int getDeltasFound() { * of {@link #getTotalDeltas()}. */ public long getTotalObjects() { - return totalObjects; + return statistics.getTotalObjects(); } /** @@ -2203,7 +2180,7 @@ public long getTotalObjects() { * @since 4.0 */ public long getBitmapIndexMisses() { - return bitmapIndexMisses; + return statistics.getBitmapIndexMisses(); } /** @@ -2211,7 +2188,7 @@ public long getBitmapIndexMisses() { * actual number of deltas if a cached pack was reused. */ public long getTotalDeltas() { - return totalDeltas; + return statistics.getTotalDeltas(); } /** @@ -2219,7 +2196,7 @@ public long getTotalDeltas() { * the output. This count includes {@link #getReusedDeltas()}. */ public long getReusedObjects() { - return reusedObjects; + return statistics.getReusedObjects(); } /** @@ -2229,7 +2206,7 @@ public long getReusedObjects() { * actual number of reused deltas if a cached pack was reused. */ public long getReusedDeltas() { - return reusedDeltas; + return statistics.getReusedDeltas(); } /** @@ -2237,7 +2214,7 @@ public long getReusedDeltas() { * header, trailer, thin pack, and reused cached pack(s). */ public long getTotalBytes() { - return totalBytes; + return statistics.getTotalBytes(); } /** @@ -2249,7 +2226,7 @@ public long getTotalBytes() { * pack header or trailer. */ public long getThinPackBytes() { - return thinPackBytes; + return statistics.getThinPackBytes(); } /** @@ -2258,17 +2235,17 @@ public long getThinPackBytes() { * @return information about this type of object in the pack. */ public ObjectType byObjectType(int typeCode) { - return objectTypes[typeCode]; + return new ObjectType(statistics.byObjectType(typeCode)); } /** @return true if the resulting pack file was a shallow pack. */ public boolean isShallow() { - return depth > 0; + return statistics.isShallow(); } /** @return depth (in commits) the pack includes if shallow. */ public int getDepth() { - return depth; + return statistics.getDepth(); } /** @@ -2277,7 +2254,7 @@ public int getDepth() { * that occur when a cached pack is selected for reuse. */ public long getTimeCounting() { - return timeCounting; + return statistics.getTimeCounting(); } /** @@ -2286,7 +2263,7 @@ public long getTimeCounting() { * can be assumed to already have. */ public long getTimeSearchingForReuse() { - return timeSearchingForReuse; + return statistics.getTimeSearchingForReuse(); } /** @@ -2296,7 +2273,7 @@ public long getTimeSearchingForReuse() { * together and improve delta compression ratios. */ public long getTimeSearchingForSizes() { - return timeSearchingForSizes; + return statistics.getTimeSearchingForSizes(); } /** @@ -2306,7 +2283,7 @@ public long getTimeSearchingForSizes() { * delta compression. */ public long getTimeCompressing() { - return timeCompressing; + return statistics.getTimeCompressing(); } /** @@ -2316,16 +2293,12 @@ public long getTimeCompressing() { * value. */ public long getTimeWriting() { - return timeWriting; + return statistics.getTimeWriting(); } /** @return total time spent processing this pack. */ public long getTimeTotal() { - return timeCounting - + timeSearchingForReuse - + timeSearchingForSizes - + timeCompressing - + timeWriting; + return statistics.getTimeTotal(); } /** @@ -2333,14 +2306,12 @@ public long getTimeTotal() { * {@code getTotalBytes() / (getTimeWriting() / 1000.0)}. */ public double getTransferRate() { - return getTotalBytes() / (getTimeWriting() / 1000.0); + return statistics.getTransferRate(); } /** @return formatted message string for display to clients. */ public String getMessage() { - return MessageFormat.format(JGitText.get().packWriterStatistics, // - Long.valueOf(totalObjects), Long.valueOf(totalDeltas), // - Long.valueOf(reusedObjects), Long.valueOf(reusedDeltas)); + return statistics.getMessage(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackStatistics.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackStatistics.java new file mode 100644 index 000000000..66ceaa667 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackStatistics.java @@ -0,0 +1,462 @@ +/* + * Copyright (C) 2015, Google Inc. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.storage.pack; + +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; +import static org.eclipse.jgit.lib.Constants.OBJ_TAG; +import static org.eclipse.jgit.lib.Constants.OBJ_TREE; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.CachedPack; +import org.eclipse.jgit.lib.ObjectId; + +/** + * Statistics about {@link org.eclipse.jgit.internal.storage.pack.PackWriter} + * pack creation. + * + * @since 4.1 + */ +public class PackStatistics { + /** + * Statistics about a single type of object (commits, tags, trees and + * blobs). + */ + public static class ObjectType { + /** + * POJO for accumulating the ObjectType statistics. + */ + public static class Accumulator { + /** Count of objects of this type. */ + public long cntObjects; + + /** Count of deltas of this type. */ + public long cntDeltas; + + /** Count of reused objects of this type. */ + public long reusedObjects; + + /** Count of reused deltas of this type. */ + public long reusedDeltas; + + /** Count of bytes for all objects of this type. */ + public long bytes; + + /** Count of delta bytes for objects of this type. */ + public long deltaBytes; + } + + private ObjectType.Accumulator objectType; + + /** + * Creates a new {@link ObjectType} object from the accumulator. + * + * @param accumulator + * the accumulator of the statistics + */ + public ObjectType(ObjectType.Accumulator accumulator) { + objectType = accumulator; + } + + /** + * @return total number of objects output. This total includes the value + * of {@link #getDeltas()}. + */ + public long getObjects() { + return objectType.cntObjects; + } + + /** + * @return total number of deltas output. This may be lower than the + * actual number of deltas if a cached pack was reused. + */ + public long getDeltas() { + return objectType.cntDeltas; + } + + /** + * @return number of objects whose existing representation was reused in + * the output. This count includes {@link #getReusedDeltas()}. + */ + public long getReusedObjects() { + return objectType.reusedObjects; + } + + /** + * @return number of deltas whose existing representation was reused in + * the output, as their base object was also output or was + * assumed present for a thin pack. This may be lower than the + * actual number of reused deltas if a cached pack was reused. + */ + public long getReusedDeltas() { + return objectType.reusedDeltas; + } + + /** + * @return total number of bytes written. This size includes the object + * headers as well as the compressed data. This size also + * includes all of {@link #getDeltaBytes()}. + */ + public long getBytes() { + return objectType.bytes; + } + + /** + * @return number of delta bytes written. This size includes the object + * headers for the delta objects. + */ + public long getDeltaBytes() { + return objectType.deltaBytes; + } + } + + /** + * POJO for accumulating the statistics. + */ + public static class Accumulator { + /** The set of objects to be included in the pack. */ + public Set interestingObjects; + + /** The set of objects to be excluded from the pack. */ + public Set uninterestingObjects; + + /** The collection of reused packs in the upload. */ + public List reusedPacks; + + /** If a shallow pack, the depth in commits. */ + public int depth; + + /** + * The count of objects in the pack that went through the delta search + * process in order to find a potential delta base. + */ + public int deltaSearchNonEdgeObjects; + + /** + * The count of objects in the pack that went through delta base search + * and found a suitable base. This is a subset of + * deltaSearchNonEdgeObjects. + */ + public int deltasFound; + + /** The total count of objects in the pack. */ + public long totalObjects; + + /** + * The count of objects that needed to be discovered through an object + * walk because they were not found in bitmap indices. + */ + public long bitmapIndexMisses; + + /** The total count of deltas output. */ + public long totalDeltas; + + /** The count of reused objects in the pack. */ + public long reusedObjects; + + /** The count of reused deltas in the pack. */ + public long reusedDeltas; + + /** The count of total bytes in the pack. */ + public long totalBytes; + + /** The size of the thin pack in bytes, if a thin pack was generated. */ + public long thinPackBytes; + + /** Time in ms spent counting the objects that will go into the pack. */ + public long timeCounting; + + /** Time in ms spent searching for objects to reuse. */ + public long timeSearchingForReuse; + + /** Time in ms spent searching for sizes of objects. */ + public long timeSearchingForSizes; + + /** Time in ms spent compressing the pack. */ + public long timeCompressing; + + /** Time in ms spent writing the pack. */ + public long timeWriting; + + /** + * Statistics about each object type in the pack (commits, tags, trees + * and blobs.) + */ + public ObjectType.Accumulator[] objectTypes; + + { + objectTypes = new ObjectType.Accumulator[5]; + objectTypes[OBJ_COMMIT] = new ObjectType.Accumulator(); + objectTypes[OBJ_TREE] = new ObjectType.Accumulator(); + objectTypes[OBJ_BLOB] = new ObjectType.Accumulator(); + objectTypes[OBJ_TAG] = new ObjectType.Accumulator(); + } + } + + private Accumulator statistics; + + /** + * Creates a new {@link PackStatistics} object from the accumulator. + * + * @param accumulator + * the accumulator of the statistics + */ + public PackStatistics(Accumulator accumulator) { + // Note: PackStatistics directly serves up the collections in the + // accumulator. + statistics = accumulator; + } + + /** + * @return unmodifiable collection of objects to be included in the pack. + * May be {@code null} if the pack was hand-crafted in a unit test. + */ + public Set getInterestingObjects() { + return statistics.interestingObjects; + } + + /** + * @return unmodifiable collection of objects that should be excluded from + * the pack, as the peer that will receive the pack already has + * these objects. + */ + public Set getUninterestingObjects() { + return statistics.uninterestingObjects; + } + + /** + * @return unmodifiable list of the cached packs that were reused in the + * output, if any were selected for reuse. + */ + public List getReusedPacks() { + return statistics.reusedPacks; + } + + /** + * @return number of objects in the output pack that went through the delta + * search process in order to find a potential delta base. + */ + public int getDeltaSearchNonEdgeObjects() { + return statistics.deltaSearchNonEdgeObjects; + } + + /** + * @return number of objects in the output pack that went through delta base + * search and found a suitable base. This is a subset of + * {@link #getDeltaSearchNonEdgeObjects()}. + */ + public int getDeltasFound() { + return statistics.deltasFound; + } + + /** + * @return total number of objects output. This total includes the value of + * {@link #getTotalDeltas()}. + */ + public long getTotalObjects() { + return statistics.totalObjects; + } + + /** + * @return the count of objects that needed to be discovered through an + * object walk because they were not found in bitmap indices. + * Returns -1 if no bitmap indices were found. + */ + public long getBitmapIndexMisses() { + return statistics.bitmapIndexMisses; + } + + /** + * @return total number of deltas output. This may be lower than the actual + * number of deltas if a cached pack was reused. + */ + public long getTotalDeltas() { + return statistics.totalDeltas; + } + + /** + * @return number of objects whose existing representation was reused in the + * output. This count includes {@link #getReusedDeltas()}. + */ + public long getReusedObjects() { + return statistics.reusedObjects; + } + + /** + * @return number of deltas whose existing representation was reused in the + * output, as their base object was also output or was assumed + * present for a thin pack. This may be lower than the actual number + * of reused deltas if a cached pack was reused. + */ + public long getReusedDeltas() { + return statistics.reusedDeltas; + } + + /** + * @return total number of bytes written. This size includes the pack + * header, trailer, thin pack, and reused cached pack(s). + */ + public long getTotalBytes() { + return statistics.totalBytes; + } + + /** + * @return size of the thin pack in bytes, if a thin pack was generated. A + * thin pack is created when the client already has objects and some + * deltas are created against those objects, or if a cached pack is + * being used and some deltas will reference objects in the cached + * pack. This size does not include the pack header or trailer. + */ + public long getThinPackBytes() { + return statistics.thinPackBytes; + } + + /** + * @param typeCode + * object type code, e.g. OBJ_COMMIT or OBJ_TREE. + * @return information about this type of object in the pack. + */ + public ObjectType byObjectType(int typeCode) { + return new ObjectType(statistics.objectTypes[typeCode]); + } + + /** @return true if the resulting pack file was a shallow pack. */ + public boolean isShallow() { + return statistics.depth > 0; + } + + /** @return depth (in commits) the pack includes if shallow. */ + public int getDepth() { + return statistics.depth; + } + + /** + * @return time in milliseconds spent enumerating the objects that need to + * be included in the output. This time includes any restarts that + * occur when a cached pack is selected for reuse. + */ + public long getTimeCounting() { + return statistics.timeCounting; + } + + /** + * @return time in milliseconds spent matching existing representations + * against objects that will be transmitted, or that the client can + * be assumed to already have. + */ + public long getTimeSearchingForReuse() { + return statistics.timeSearchingForReuse; + } + + /** + * @return time in milliseconds spent finding the sizes of all objects that + * will enter the delta compression search window. The sizes need to + * be known to better match similar objects together and improve + * delta compression ratios. + */ + public long getTimeSearchingForSizes() { + return statistics.timeSearchingForSizes; + } + + /** + * @return time in milliseconds spent on delta compression. This is observed + * wall-clock time and does not accurately track CPU time used when + * multiple threads were used to perform the delta compression. + */ + public long getTimeCompressing() { + return statistics.timeCompressing; + } + + /** + * @return time in milliseconds spent writing the pack output, from start of + * header until end of trailer. The transfer speed can be + * approximated by dividing {@link #getTotalBytes()} by this value. + */ + public long getTimeWriting() { + return statistics.timeWriting; + } + + /** @return total time spent processing this pack. */ + public long getTimeTotal() { + return statistics.timeCounting + statistics.timeSearchingForReuse + + statistics.timeSearchingForSizes + statistics.timeCompressing + + statistics.timeWriting; + } + + /** + * @return get the average output speed in terms of bytes-per-second. + * {@code getTotalBytes() / (getTimeWriting() / 1000.0)}. + */ + public double getTransferRate() { + return getTotalBytes() / (getTimeWriting() / 1000.0); + } + + /** @return formatted message string for display to clients. */ + public String getMessage() { + return MessageFormat.format(JGitText.get().packWriterStatistics, + Long.valueOf(statistics.totalObjects), + Long.valueOf(statistics.totalDeltas), + Long.valueOf(statistics.reusedObjects), + Long.valueOf(statistics.reusedDeltas)); + } + + /** @return a map containing ObjectType statistics. */ + public Map getObjectTypes() { + HashMap map = new HashMap<>(); + map.put(Integer.valueOf(OBJ_BLOB), new ObjectType( + statistics.objectTypes[OBJ_BLOB])); + map.put(Integer.valueOf(OBJ_COMMIT), new ObjectType( + statistics.objectTypes[OBJ_COMMIT])); + map.put(Integer.valueOf(OBJ_TAG), new ObjectType( + statistics.objectTypes[OBJ_TAG])); + map.put(Integer.valueOf(OBJ_TREE), new ObjectType( + statistics.objectTypes[OBJ_TREE])); + return map; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHook.java new file mode 100644 index 000000000..53eeab1df --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHook.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2015, Google Inc. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.transport; + +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.storage.pack.PackStatistics; + +/** + * Hook invoked by {@link UploadPack} after the pack has been uploaded. + *

+ * Implementors of the interface are responsible for associating the current + * thread to a particular connection, if they need to also include connection + * information. One method is to use a {@link java.lang.ThreadLocal} to remember + * the connection information before invoking UploadPack. + * + * @since 4.1 + */ +public interface PostUploadHook { + /** A simple no-op hook. */ + public static final PostUploadHook NULL = new PostUploadHook() { + public void onPostUpload(PackStatistics stats) { + // Do nothing. + } + }; + + /** + * Notifies the hook that a pack has been sent. + * + * @param stats + * the statistics gathered by {@link PackWriter} for the uploaded + * pack + */ + public void onPostUpload(PackStatistics stats); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHookChain.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHookChain.java new file mode 100644 index 000000000..4e2eaeaff --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PostUploadHookChain.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2015, Google Inc. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.transport; + +import java.util.List; + +import org.eclipse.jgit.storage.pack.PackStatistics; + +/** + * {@link PostUploadHook} that delegates to a list of other hooks. + *

+ * Hooks are run in the order passed to the constructor. + * + * @since 4.1 + */ +public class PostUploadHookChain implements PostUploadHook { + private final PostUploadHook[] hooks; + private final int count; + + /** + * Create a new hook chaining the given hooks together. + * + * @param hooks + * hooks to execute, in order. + * @return a new chain of the given hooks. + */ + public static PostUploadHook newChain(List hooks) { + PostUploadHook[] newHooks = new PostUploadHook[hooks.size()]; + int i = 0; + for (PostUploadHook hook : hooks) + if (hook != PostUploadHook.NULL) + newHooks[i++] = hook; + if (i == 0) + return PostUploadHook.NULL; + else if (i == 1) + return newHooks[0]; + else + return new PostUploadHookChain(newHooks, i); + } + + public void onPostUpload(PackStatistics stats) { + for (int i = 0; i < count; i++) + hooks[i].onPostUpload(stats); + } + + private PostUploadHookChain(PostUploadHook[] hooks, int count) { + this.hooks = hooks; + this.count = count; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 1e6aabbe0..4ff5b789c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -94,6 +94,7 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter; import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck; import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser; import org.eclipse.jgit.util.io.InterruptTimer; @@ -253,6 +254,9 @@ public Set getOptions() { /** Hook handling the various upload phases. */ private PreUploadHook preUploadHook = PreUploadHook.NULL; + /** Hook for taking post upload actions. */ + private PostUploadHook postUploadHook = PostUploadHook.NULL; + /** Capabilities requested by the client. */ private Set options; String userAgent; @@ -306,7 +310,7 @@ public Set getOptions() { private boolean noDone; - private PackWriter.Statistics statistics; + private PackStatistics statistics; private UploadPackLogger logger = UploadPackLogger.NULL; @@ -519,7 +523,7 @@ public void setRefFilter(final RefFilter refFilter) { this.refFilter = refFilter != null ? refFilter : RefFilter.DEFAULT; } - /** @return the configured upload hook. */ + /** @return the configured pre upload hook. */ public PreUploadHook getPreUploadHook() { return preUploadHook; } @@ -534,6 +538,25 @@ public void setPreUploadHook(PreUploadHook hook) { preUploadHook = hook != null ? hook : PreUploadHook.NULL; } + /** + * @return the configured post upload hook. + * @since 4.1 + */ + public PostUploadHook getPostUploadHook() { + return postUploadHook; + } + + /** + * Set the hook for post upload actions (logging, repacking). + * + * @param hook + * the hook; if null no special actions are taken. + * @since 4.1 + */ + public void setPostUploadHook(PostUploadHook hook) { + postUploadHook = hook != null ? hook : PostUploadHook.NULL; + } + /** * Set the configuration used by the pack generator. * @@ -562,7 +585,12 @@ public void setTransferConfig(TransferConfig tc) { } } - /** @return the configured logger. */ + /** + * @return the configured logger. + * + * @deprecated Use {@link #getPreUploadHook()}. + */ + @Deprecated public UploadPackLogger getLogger() { return logger; } @@ -572,7 +600,9 @@ public UploadPackLogger getLogger() { * * @param logger * the logger instance. If null, no logging occurs. + * @deprecated Use {@link #setPreUploadHook(PreUploadHook)}. */ + @Deprecated public void setLogger(UploadPackLogger logger) { this.logger = logger; } @@ -654,8 +684,23 @@ public void upload(final InputStream input, final OutputStream output, * was sent, such as during the negotation phase of a smart HTTP * connection, or if the client was already up-to-date. * @since 3.0 + * @deprecated Use {@link #getStatistics()}. */ + @Deprecated public PackWriter.Statistics getPackStatistics() { + return statistics == null ? null + : new PackWriter.Statistics(statistics); + } + + /** + * Get the PackWriter's statistics if a pack was sent to the client. + * + * @return statistics about pack output, if a pack was sent. Null if no pack + * was sent, such as during the negotation phase of a smart HTTP + * connection, or if the client was already up-to-date. + * @since 4.1 + */ + public PackStatistics getStatistics() { return statistics; } @@ -1468,8 +1513,10 @@ else if (ref.getName().startsWith(Constants.R_HEADS)) } finally { statistics = pw.getStatistics(); - if (statistics != null) - logger.onPackStatistics(statistics); + if (statistics != null) { + postUploadHook.onPostUpload(statistics); + logger.onPackStatistics(new PackWriter.Statistics(statistics)); + } pw.close(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java index 99fa6e02b..85ebecc45 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java @@ -52,7 +52,10 @@ * thread to a particular connection, if they need to also include connection * information. One method is to use a {@link java.lang.ThreadLocal} to remember * the connection information before invoking UploadPack. + * + * @deprecated use {@link PostUploadHook} instead */ +@Deprecated public interface UploadPackLogger { /** A simple no-op logger. */ public static final UploadPackLogger NULL = new UploadPackLogger() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLoggerChain.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLoggerChain.java index 3f14cc6f5..4ea0319d9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLoggerChain.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLoggerChain.java @@ -48,10 +48,13 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; /** - * {@link UploadPackLogger} that delegates to a list of other loggers. + * UploadPackLogger that delegates to a list of other loggers. *

* loggers are run in the order passed to the constructor. + * + * @deprecated Use {@link PostUploadHookChain} instead. */ +@Deprecated public class UploadPackLoggerChain implements UploadPackLogger { private final UploadPackLogger[] loggers; private final int count;