From 8803718493b3e5271b2acbf7c74d6f200dd65929 Mon Sep 17 00:00:00 2001 From: Zhen Chen Date: Tue, 22 Nov 2016 15:53:06 -0800 Subject: [PATCH] dump HTTP: Avoid being confused by Content-Length of a gzipped stream TransportHttp sets 'Accept-Encoding: gzip' to allow the server to compress HTTP responses. When fetching a loose object over HTTP, it uses the following code to read the response: InputStream in = openInputStream(c); int len = c.getContentLength(); return new FileStream(in, len); If the content is gzipped, openInputStream decompresses it and produces the correct content for the object. Unfortunately the Content-Length header contains the length of the compressed stream instead of the actual content length. Use a length of -1 instead since we don't know the actual length. Loose objects are already compressed, so the gzip encoding typically produces a longer compressed payload. The value from the Content-Length is too high, producing EOFException: Short read of block. Change-Id: I8d5284dad608e3abd8217823da2b365e8cd998b0 Signed-off-by: Zhen Chen Helped-by: Jonathan Nieder --- .../eclipse/jgit/http/server/ServletUtils.java | 3 ++- .../eclipse/jgit/transport/TransportHttp.java | 18 +++++++++++++++--- .../src/org/eclipse/jgit/util/HttpSupport.java | 3 +++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java index 042ccf3dd..1336d6e93 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.http.server; import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; +import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING; import static org.eclipse.jgit.util.HttpSupport.HDR_ETAG; @@ -111,7 +112,7 @@ public static InputStream getInputStream(final HttpServletRequest req) throws IOException { InputStream in = req.getInputStream(); final String enc = req.getHeader(HDR_CONTENT_ENCODING); - if (ENCODING_GZIP.equals(enc) || "x-gzip".equals(enc)) //$NON-NLS-1$ + if (ENCODING_GZIP.equals(enc) || ENCODING_X_GZIP.equals(enc)) //$NON-NLS-1$ in = new GZIPInputStream(in); else if (enc != null) throw new IOException(MessageFormat.format(HttpServerText.get().encodingNotSupportedByThisLibrary diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 1166080f2..eb2d62cc3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.transport; import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; +import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING; @@ -575,7 +576,7 @@ protected HttpConnection httpOpen(String method, URL u) final InputStream openInputStream(HttpConnection conn) throws IOException { InputStream input = conn.getInputStream(); - if (ENCODING_GZIP.equals(conn.getHeaderField(HDR_CONTENT_ENCODING))) + if (isGzipContent(conn)) input = new GZIPInputStream(input); return input; } @@ -591,6 +592,11 @@ private boolean isSmartHttp(final HttpConnection c, final String service) { return expType.equals(actType); } + private boolean isGzipContent(final HttpConnection c) { + return ENCODING_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING)) + || ENCODING_X_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING)); + } + private void readSmartHeaders(final InputStream in, final String service) throws IOException { // A smart reply will have a '#' after the first 4 bytes, but @@ -685,8 +691,14 @@ FileStream open(final String path) throws IOException { switch (HttpSupport.response(c)) { case HttpConnection.HTTP_OK: final InputStream in = openInputStream(c); - final int len = c.getContentLength(); - return new FileStream(in, len); + // If content is being gzipped and then transferred, the content + // length in the header is the zipped content length, not the + // actual content length. + if (!isGzipContent(c)) { + final int len = c.getContentLength(); + return new FileStream(in, len); + } + return new FileStream(in); case HttpConnection.HTTP_NOT_FOUND: throw new FileNotFoundException(u.toString()); default: diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java index 202645b9d..fbb506e5a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java @@ -144,6 +144,9 @@ public class HttpSupport { /** The {@code gzip} encoding value for {@link #HDR_ACCEPT_ENCODING}. */ public static final String ENCODING_GZIP = "gzip"; //$NON-NLS-1$ + /** The {@code x-gzip} encoding value for {@link #HDR_ACCEPT_ENCODING}. */ + public static final String ENCODING_X_GZIP = "x-gzip"; //$NON-NLS-1$ + /** The standard {@code text/plain} MIME type. */ public static final String TEXT_PLAIN = "text/plain"; //$NON-NLS-1$