From 64b524e3ca3d1f66edaa49eda2d8863ddca779b5 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 16 Mar 2011 13:46:53 -0700 Subject: [PATCH] UploadPack: Add a PreUploadHook to monitor and control behavior Embedding applications can use this hook to watch actions within UploadPack and possibly reject them. This could be useful to prevent clones of a large repository from this server, or to stop abusive negotiation rounds that offer thousands of objects in a single batch. Change-Id: Id96f1885ac4d61f22c80b6418fff54184b7348ba Signed-off-by: Shawn O. Pearce --- .../http/server/SmartServiceInfoRefs.java | 10 +- .../jgit/http/server/UploadPackServlet.java | 7 + .../eclipse/jgit/transport/PreUploadHook.java | 158 ++++++++++++++++++ .../eclipse/jgit/transport/UploadPack.java | 124 +++++++++++--- .../UploadPackMayNotContinueException.java | 77 +++++++++ 5 files changed, 354 insertions(+), 22 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/transport/PreUploadHook.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackMayNotContinueException.java diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/SmartServiceInfoRefs.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/SmartServiceInfoRefs.java index 7152c88ed..935867cef 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/SmartServiceInfoRefs.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/SmartServiceInfoRefs.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.http.server; import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE; import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED; import static org.eclipse.jgit.http.server.ServletUtils.ATTRIBUTE_HANDLER; import static org.eclipse.jgit.http.server.ServletUtils.getRepository; @@ -63,6 +64,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.PacketLineOut; import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser; +import org.eclipse.jgit.transport.UploadPackMayNotContinueException; import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException; import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; @@ -119,10 +121,10 @@ private void service(ServletRequest request, ServletResponse response) throws IOException { final HttpServletRequest req = (HttpServletRequest) request; final HttpServletResponse rsp = (HttpServletResponse) response; + final SmartOutputStream buf = new SmartOutputStream(req, rsp); try { rsp.setContentType("application/x-" + svc + "-advertisement"); - final SmartOutputStream buf = new SmartOutputStream(req, rsp); final PacketLineOut out = new PacketLineOut(buf); out.writeString("# service=" + svc + "\n"); out.end(); @@ -133,6 +135,12 @@ private void service(ServletRequest request, ServletResponse response) } catch (ServiceNotEnabledException e) { rsp.sendError(SC_FORBIDDEN); + + } catch (UploadPackMayNotContinueException e) { + if (e.isOutput()) + buf.close(); + else + rsp.sendError(SC_SERVICE_UNAVAILABLE); } } diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/UploadPackServlet.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/UploadPackServlet.java index 192adc56b..e60c5068c 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/UploadPackServlet.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/UploadPackServlet.java @@ -45,6 +45,7 @@ import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE; import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED; import static javax.servlet.http.HttpServletResponse.SC_UNSUPPORTED_MEDIA_TYPE; import static org.eclipse.jgit.http.server.ServletUtils.ATTRIBUTE_HANDLER; @@ -67,6 +68,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser; import org.eclipse.jgit.transport.UploadPack; +import org.eclipse.jgit.transport.UploadPackMayNotContinueException; import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException; import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; import org.eclipse.jgit.transport.resolver.UploadPackFactory; @@ -171,6 +173,11 @@ public void flush() throws IOException { up.upload(getInputStream(req), out, null); out.close(); + } catch (UploadPackMayNotContinueException e) { + if (!e.isOutput()) + rsp.sendError(SC_SERVICE_UNAVAILABLE); + return; + } catch (IOException e) { getServletContext().log(HttpServerText.get().internalErrorDuringUploadPack, e); rsp.reset(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PreUploadHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PreUploadHook.java new file mode 100644 index 000000000..e37d80bde --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PreUploadHook.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2011, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.transport; + +import java.util.Collection; + +import org.eclipse.jgit.lib.ObjectId; + +/** + * Hook invoked by {@link UploadPack} before during critical phases. + *

+ * If any hook function throws {@link UploadPackMayNotContinueException} then + * processing stops immediately and the exception is thrown up the call stack. + * Most phases of UploadPack will try to report the exception's message text to + * the end-user over the client's protocol connection. + */ +public interface PreUploadHook { + /** A simple no-op hook. */ + public static final PreUploadHook NULL = new PreUploadHook() { + public void onPreAdvertiseRefs(UploadPack up) + throws UploadPackMayNotContinueException { + // Do nothing. + } + + public void onBeginNegotiateRound(UploadPack up, + Collection wants, int cntOffered) + throws UploadPackMayNotContinueException { + // Do nothing. + } + + public void onEndNegotiateRound(UploadPack up, + Collection wants, int cntCommon, + int cntNotFound, boolean ready) + throws UploadPackMayNotContinueException { + // Do nothing. + } + + public void onSendPack(UploadPack up, + Collection wants, + Collection haves) + throws UploadPackMayNotContinueException { + // Do nothing. + } + }; + + /** + * Invoked just before {@link UploadPack#sendAdvertisedRefs(RefAdvertiser)}. + * + * @param up + * the upload pack instance handling the connection. + * @throws UploadPackMayNotContinueException + * abort; the message will be sent to the user. + */ + public void onPreAdvertiseRefs(UploadPack up) + throws UploadPackMayNotContinueException; + + /** + * Invoked before negotiation round is started. + * + * @param up + * the upload pack instance handling the connection. + * @param wants + * the list of wanted objects. + * @param cntOffered + * number of objects the client has offered. + * @throws UploadPackMayNotContinueException + * abort; the message will be sent to the user. + */ + public void onBeginNegotiateRound(UploadPack up, + Collection wants, int cntOffered) + throws UploadPackMayNotContinueException; + + /** + * Invoked after a negotiation round is completed. + * + * @param up + * the upload pack instance handling the connection. + * @param wants + * the list of wanted objects. + * @param cntCommon + * number of objects this round found to be common. In a smart + * HTTP transaction this includes the objects that were + * previously found to be common. + * @param cntNotFound + * number of objects in this round the local repository does not + * have, but that were offered as potential common bases. + * @param ready + * true if a pack is ready to be sent (the commit graph was + * successfully cut). + * @throws UploadPackMayNotContinueException + * abort; the message will be sent to the user. + */ + public void onEndNegotiateRound(UploadPack up, + Collection wants, int cntCommon, + int cntNotFound, boolean ready) + throws UploadPackMayNotContinueException; + + /** + * Invoked just before a pack will be sent to the client. + * + * @param up + * the upload pack instance handling the connection. + * @param wants + * the list of wanted objects. These may be RevObject or + * RevCommit if the processed parsed them. Implementors should + * not rely on the values being parsed. + * @param haves + * the list of common objects. Empty on an initial clone request. + * These may be RevObject or RevCommit if the processed parsed + * them. Implementors should not rely on the values being parsed. + * @throws UploadPackMayNotContinueException + * abort; the message will be sent to the user. + */ + public void onSendPack(UploadPack up, Collection wants, + Collection haves) + throws UploadPackMayNotContinueException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 4b225575e..2802c0712 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -142,6 +142,9 @@ public class UploadPack { /** Filter used while advertising the refs to the client. */ private RefFilter refFilter; + /** Hook handling the various upload phases. */ + private PreUploadHook preUploadHook = PreUploadHook.NULL; + /** Capabilities requested by the client. */ private final Set options = new HashSet(); @@ -220,6 +223,9 @@ public final RevWalk getRevWalk() { /** @return all refs which were advertised to the client. */ public final Map getAdvertisedRefs() { + if (refs == null) { + refs = refFilter.filter(db.getAllRefs()); + } return refs; } @@ -281,6 +287,21 @@ public void setRefFilter(final RefFilter refFilter) { this.refFilter = refFilter != null ? refFilter : RefFilter.DEFAULT; } + /** @return the configured upload hook. */ + public PreUploadHook getPreUploadHook() { + return preUploadHook; + } + + /** + * Set the hook that controls how this instance will behave. + * + * @param hook + * the hook; if null no special actions are taken. + */ + public void setPreUploadHook(PreUploadHook hook) { + preUploadHook = hook != null ? hook : PreUploadHook.NULL; + } + /** * Set the configuration used by the pack generator. * @@ -367,16 +388,18 @@ private void service() throws IOException { sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut)); else { advertised = new HashSet(); - refs = refFilter.filter(db.getAllRefs()); - for (Ref ref : refs.values()) { + for (Ref ref : getAdvertisedRefs().values()) { if (ref.getObjectId() != null) advertised.add(ref.getObjectId()); } } recvWants(); - if (wantIds.isEmpty()) + if (wantIds.isEmpty()) { + preUploadHook.onBeginNegotiateRound(this, wantIds, 0); + preUploadHook.onEndNegotiateRound(this, wantIds, 0, 0, false); return; + } if (options.contains(OPTION_MULTI_ACK_DETAILED)) multiAck = MultiAck.DETAILED; @@ -396,8 +419,21 @@ else if (options.contains(OPTION_MULTI_ACK)) * the advertisement formatter. * @throws IOException * the formatter failed to write an advertisement. + * @throws UploadPackMayNotContinueException + * the hook denied advertisement. */ - public void sendAdvertisedRefs(final RefAdvertiser adv) throws IOException { + public void sendAdvertisedRefs(final RefAdvertiser adv) throws IOException, + UploadPackMayNotContinueException { + try { + preUploadHook.onPreAdvertiseRefs(this); + } catch (UploadPackMayNotContinueException fail) { + if (fail.getMessage() != null) { + adv.writeOne("ERR " + fail.getMessage()); + fail.setOutput(); + } + throw fail; + } + adv.init(db); adv.advertiseCapability(OPTION_INCLUDE_TAG); adv.advertiseCapability(OPTION_MULTI_ACK_DETAILED); @@ -408,8 +444,7 @@ public void sendAdvertisedRefs(final RefAdvertiser adv) throws IOException { adv.advertiseCapability(OPTION_THIN_PACK); adv.advertiseCapability(OPTION_NO_PROGRESS); adv.setDerefTags(true); - refs = refFilter.filter(db.getAllRefs()); - advertised = adv.send(refs); + advertised = adv.send(getAdvertisedRefs()); adv.end(); } @@ -487,6 +522,16 @@ else if (multiAck != MultiAck.OFF) private ObjectId processHaveLines(List peerHas, ObjectId last) throws IOException { + try { + preUploadHook.onBeginNegotiateRound(this, wantIds, peerHas.size()); + } catch (UploadPackMayNotContinueException fail) { + if (fail.getMessage() != null) { + pckOut.writeString("ERR " + fail.getMessage() + "\n"); + fail.setOutput(); + } + throw fail; + } + if (peerHas.isEmpty()) return last; @@ -504,6 +549,7 @@ private ObjectId processHaveLines(List peerHas, ObjectId last) needMissing = true; } + int haveCnt = 0; AsyncRevObjectQueue q = walk.parseAny(toParse, needMissing); try { for (;;) { @@ -557,6 +603,7 @@ private ObjectId processHaveLines(List peerHas, ObjectId last) } last = obj; + haveCnt++; if (obj instanceof RevCommit) { RevCommit c = (RevCommit) obj; @@ -590,35 +637,52 @@ private ObjectId processHaveLines(List peerHas, ObjectId last) } finally { q.release(); } + int missCnt = peerHas.size() - haveCnt; // If we don't have one of the objects but we're also willing to // create a pack at this point, let the client know so it stops // telling us about its history. // boolean didOkToGiveUp = false; - for (int i = peerHas.size() - 1; i >= 0; i--) { - ObjectId id = peerHas.get(i); - if (walk.lookupOrNull(id) == null) { - didOkToGiveUp = true; - if (okToGiveUp()) { - switch (multiAck) { - case OFF: - break; - case CONTINUE: - pckOut.writeString("ACK " + id.name() + " continue\n"); - break; - case DETAILED: - pckOut.writeString("ACK " + id.name() + " ready\n"); - break; + boolean sentReady = false; + if (0 < missCnt) { + for (int i = peerHas.size() - 1; i >= 0; i--) { + ObjectId id = peerHas.get(i); + if (walk.lookupOrNull(id) == null) { + didOkToGiveUp = true; + if (okToGiveUp()) { + switch (multiAck) { + case OFF: + break; + case CONTINUE: + pckOut.writeString("ACK " + id.name() + " continue\n"); + break; + case DETAILED: + pckOut.writeString("ACK " + id.name() + " ready\n"); + sentReady = true; + break; + } } + break; } - break; } } if (multiAck == MultiAck.DETAILED && !didOkToGiveUp && okToGiveUp()) { ObjectId id = peerHas.get(peerHas.size() - 1); pckOut.writeString("ACK " + id.name() + " ready\n"); + sentReady = true; + } + + try { + preUploadHook.onEndNegotiateRound(this, wantAll, // + haveCnt, missCnt, sentReady); + } catch (UploadPackMayNotContinueException fail) { + if (fail.getMessage() != null) { + pckOut.writeString("ERR " + fail.getMessage() + "\n"); + fail.setOutput(); + } + throw fail; } peerHas.clear(); @@ -697,6 +761,24 @@ private void sendPack() throws IOException { } } + try { + if (wantAll.isEmpty()) { + preUploadHook.onSendPack(this, wantIds, commonBase); + } else { + preUploadHook.onSendPack(this, wantAll, commonBase); + } + } catch (UploadPackMayNotContinueException noPack) { + if (sideband && noPack.getMessage() != null) { + noPack.setOutput(); + SideBandOutputStream err = new SideBandOutputStream( + SideBandOutputStream.CH_ERROR, + SideBandOutputStream.SMALL_BUF, rawOut); + err.write(Constants.encode(noPack.getMessage())); + err.flush(); + } + throw noPack; + } + PackConfig cfg = packConfig; if (cfg == null) cfg = new PackConfig(db); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackMayNotContinueException.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackMayNotContinueException.java new file mode 100644 index 000000000..c2395f37d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackMayNotContinueException.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2011, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.transport; + +import java.io.IOException; + +/** Indicates UploadPack may not continue execution. */ +public class UploadPackMayNotContinueException extends IOException { + private static final long serialVersionUID = 1L; + + private boolean output; + + /** Initialize with no message. */ + public UploadPackMayNotContinueException() { + // Do not set a message. + } + + /** + * @param msg + * a message explaining why it cannot continue. This message may + * be shown to an end-user. + */ + public UploadPackMayNotContinueException(String msg) { + super(msg); + } + + /** @return true if the message was already output to the client. */ + public boolean isOutput() { + return output; + } + + /** Mark this message has being sent to the client. */ + public void setOutput() { + output = true; + } +}