motiejus
/
jgit
1
Fork 0
Code Releases Activity

Merge branch 'stable-5.0' 

* stable-5.0:
  Teach UploadPack "filter" in protocol v2 fetch
  Refactor test of capabilities output
  Refactor v2 advertisement into own function
  Refactor parsing of "filter" into its own method
  Disallow unknown args to "fetch" in protocol v2
  Teach UploadPack shallow fetch in protocol v2
  Refactor unshallowCommits to local variable
  Add protocol v2 support in http
  Give info/refs services more control over response

Change-Id: I1683902222e076e1091795e94790a264550afb7b
Signed-off-by: Jonathan Nieder <jrn@google.com>
This commit is contained in:
Jonathan Nieder 2018-06-04 22:22:24 -07:00
parent 61e2414ef7 7dbd2bfe7e
commit 903432ef4d
11 changed files with 597 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/SmartServiceInfoRefs.java
View File

@ -133,9 +133,7 @@ private void service(ServletRequest request, ServletResponse response)
res.setContentType(infoRefsResultType(svc));
final PacketLineOut out = new PacketLineOut(buf);
out.writeString("# service=" + svc + "\n");
out.end();
advertise(req, new PacketLineOutRefAdvertiser(out));
respond(req, out, svc);
buf.close();
} catch (ServiceNotAuthorizedException e) {
res.sendError(SC_UNAUTHORIZED, e.getMessage());
@ -178,6 +176,37 @@ protected abstract void advertise(HttpServletRequest req,
PacketLineOutRefAdvertiser pck) throws IOException,
ServiceNotEnabledException, ServiceNotAuthorizedException;
/**
* Writes the appropriate response to an info/refs request received by
* a smart service. In protocol v0, this starts with "#
* service=serviceName" followed by a flush packet, but this is not
* necessarily the case in other protocol versions.
* <p>
* The default implementation writes "# service=serviceName" and a
* flush packet, then calls {@link #advertise}. Subclasses should
* override this method if they support protocol versions other than
* protocol v0.
*
* @param req
* request
* @param pckOut
* destination of response
* @param serviceName
* service name to be written out in protocol v0; may or may
* not be used in other versions
* @throws IOException
* @throws ServiceNotEnabledException
* @throws ServiceNotAuthorizedException
*/
protected void respond(HttpServletRequest req,
PacketLineOut pckOut, String serviceName)
throws IOException, ServiceNotEnabledException,
ServiceNotAuthorizedException {
pckOut.writeString("# service=" + svc + '\n'); //$NON-NLS-1$
pckOut.end();
advertise(req, new PacketLineOutRefAdvertiser(pckOut));
}
private class Chain implements FilterChain {
private int filterIdx;

20
org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/UploadPackServlet.java
View File

@ -76,6 +76,7 @@
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.InternalHttpServerGlue;
import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser;
import org.eclipse.jgit.transport.ServiceMayNotContinueException;
import org.eclipse.jgit.transport.UploadPack;
@ -117,6 +118,25 @@ protected void advertise(HttpServletRequest req,
up.setBiDirectionalPipe(false);
up.sendAdvertisedRefs(pck);
} finally {
// TODO(jonathantanmy): Move responsibility for closing the
// RevWalk to UploadPack, either by making it AutoCloseable
// or by making sendAdvertisedRefs clean up after itself.
up.getRevWalk().close();
}
}
@Override
protected void respond(HttpServletRequest req,
PacketLineOut pckOut, String serviceName) throws IOException,
ServiceNotEnabledException, ServiceNotAuthorizedException {
UploadPack up = (UploadPack) req.getAttribute(ATTRIBUTE_HANDLER);
try {
up.setBiDirectionalPipe(false);
up.sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut), serviceName);
} finally {
// TODO(jonathantanmy): Move responsibility for closing the
// RevWalk to UploadPack, either by making it AutoCloseable
// or by making sendAdvertisedRefs clean up after itself.
up.getRevWalk().close();
}
}

15
org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/resolver/DefaultUploadPackFactory.java
View File

@ -43,6 +43,8 @@
package org.eclipse.jgit.http.server.resolver;
import java.util.Arrays;
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jgit.lib.Config;
@ -73,9 +75,16 @@ private static class ServiceConfig {
@Override
public UploadPack create(HttpServletRequest req, Repository db)
throws ServiceNotEnabledException, ServiceNotAuthorizedException {
if (db.getConfig().get(ServiceConfig::new).enabled)
return new UploadPack(db);
else
if (db.getConfig().get(ServiceConfig::new).enabled) {
UploadPack up = new UploadPack(db);
String header = req.getHeader("Git-Protocol"); //$NON-NLS-1$
if (header != null) {
String[] params = header.split(":"); //$NON-NLS-1$
up.setExtraParameters(Arrays.asList(params));
}
return up;
} else {
throw new ServiceNotEnabledException();
}
}
}

5
org.eclipse.jgit.http.test/META-INF/MANIFEST.MF
View File

@ -9,8 +9,8 @@ Bundle-Localization: plugin
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
Import-Package: javax.servlet;version="[2.5.0,3.2.0)",
javax.servlet.http;version="[2.5.0,3.2.0)",
org.apache.commons.codec;version="[1.6.0, 2.0.0)",
org.apache.commons.codec.binary;version="[1.6.0, 2.0.0)",
org.apache.commons.codec;version="[1.6.0,2.0.0)",
org.apache.commons.codec.binary;version="[1.6.0,2.0.0)",
org.eclipse.jetty.continuation;version="[9.4.5,10.0.0)",
org.eclipse.jetty.http;version="[9.4.5,10.0.0)",
org.eclipse.jetty.io;version="[9.4.5,10.0.0)",
@ -44,6 +44,7 @@ Import-Package: javax.servlet;version="[2.5.0,3.2.0)",
org.eclipse.jgit.transport.http.apache;version="[5.1.0,5.2.0)",
org.eclipse.jgit.transport.resolver;version="[5.1.0,5.2.0)",
org.eclipse.jgit.util;version="[5.1.0,5.2.0)",
org.hamcrest;version="[1.1.0,2.0.0)",
org.hamcrest.core;version="[1.1.0,2.0.0)",
org.junit;version="[4.12,5.0.0)",
org.junit.runner;version="[4.12,5.0.0)",

7
org.eclipse.jgit.http.test/pom.xml
View File

@ -83,6 +83,13 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<scope>test</scope>
<version>[1.1.0,2.0.0)</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit.junit.http</artifactId>

87
org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java
View File

@ -43,15 +43,20 @@
package org.eclipse.jgit.http.test;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.theInstance;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.OutputStream;
import java.net.URI;
import java.net.URL;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
@ -75,9 +80,13 @@
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.FetchConnection;
import org.eclipse.jgit.transport.PacketLineIn;
import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.transport.Transport;
import org.eclipse.jgit.transport.URIish;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.http.HttpConnection;
import org.eclipse.jgit.transport.http.JDKHttpConnectionFactory;
import org.eclipse.jgit.transport.resolver.RepositoryResolver;
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException;
import org.junit.Before;
@ -345,4 +354,82 @@ public void testListRemoteWithoutLocalRepository() throws Exception {
assertNotNull(head);
}
}
@Test
public void testHttpClientWantsV2ButServerNotConfigured() throws Exception {
JDKHttpConnectionFactory f = new JDKHttpConnectionFactory();
String url = smartAuthNoneURI.toString() + "/info/refs?service=git-upload-pack";
HttpConnection c = f.create(new URL(url));
c.setRequestMethod("GET");
c.setRequestProperty("Git-Protocol", "version=2");
c.connect();
assertThat(c.getResponseCode(), is(200));
PacketLineIn pckIn = new PacketLineIn(c.getInputStream());
// Check that we get a v0 response.
assertThat(pckIn.readString(), is("# service=git-upload-pack"));
assertThat(pckIn.readString(), theInstance(PacketLineIn.END));
assertTrue(pckIn.readString().matches("[0-9a-f]{40} HEAD.*"));
}
@Test
public void testV2HttpFirstResponse() throws Exception {
remoteRepository.getRepository().getConfig().setInt(
"protocol", null, "version", 2);
JDKHttpConnectionFactory f = new JDKHttpConnectionFactory();
String url = smartAuthNoneURI.toString() + "/info/refs?service=git-upload-pack";
HttpConnection c = f.create(new URL(url));
c.setRequestMethod("GET");
c.setRequestProperty("Git-Protocol", "version=2");
c.connect();
assertThat(c.getResponseCode(), is(200));
PacketLineIn pckIn = new PacketLineIn(c.getInputStream());
assertThat(pckIn.readString(), is("version 2"));
// What remains are capabilities - ensure that all of them are
// non-empty strings, and that we see END at the end.
String s;
while ((s = pckIn.readString()) != PacketLineIn.END) {
assertTrue(!s.isEmpty());
}
}
@Test
public void testV2HttpSubsequentResponse() throws Exception {
remoteRepository.getRepository().getConfig().setInt(
"protocol", null, "version", 2);
JDKHttpConnectionFactory f = new JDKHttpConnectionFactory();
String url = smartAuthNoneURI.toString() + "/git-upload-pack";
HttpConnection c = f.create(new URL(url));
c.setRequestMethod("POST");
c.setRequestProperty("Content-Type", "application/x-git-upload-pack-request");
c.setRequestProperty("Git-Protocol", "version=2");
c.setDoOutput(true);
c.connect();
// Test ls-refs to verify that everything is connected
// properly. Tests for other commands go in
// UploadPackTest.java.
OutputStream os = c.getOutputStream();
PacketLineOut pckOut = new PacketLineOut(os);
pckOut.writeString("command=ls-refs");
pckOut.writeDelim();
pckOut.end();
os.close();
PacketLineIn pckIn = new PacketLineIn(c.getInputStream());
// Just check that we get what looks like a ref advertisement.
String s;
while ((s = pckIn.readString()) != PacketLineIn.END) {
assertTrue(s.matches("[0-9a-f]{40} [A-Za-z/]*"));
}
assertThat(c.getResponseCode(), is(200));
}
}

221
org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java
View File

@ -15,6 +15,7 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.storage.dfs.DfsGarbageCollector;
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
@ -335,12 +336,12 @@ public UploadPack create(Object req, Repository db)
}
/*
* Invokes UploadPack with protocol v2 and sends it the given lines.
* Returns UploadPack's output stream, not including the capability
* advertisement by the server.
* Invokes UploadPack with protocol v2 and sends it the given lines,
* and returns UploadPack's output stream.
*/
private ByteArrayInputStream uploadPackV2(RequestPolicy requestPolicy,
private ByteArrayInputStream uploadPackV2Setup(RequestPolicy requestPolicy,
RefFilter refFilter, String... inputLines) throws Exception {
ByteArrayOutputStream send = new ByteArrayOutputStream();
PacketLineOut pckOut = new PacketLineOut(send);
for (String line : inputLines) {
@ -364,10 +365,37 @@ private ByteArrayInputStream uploadPackV2(RequestPolicy requestPolicy,
ByteArrayOutputStream recv = new ByteArrayOutputStream();
up.upload(new ByteArrayInputStream(send.toByteArray()), recv, null);
ByteArrayInputStream recvStream = new ByteArrayInputStream(recv.toByteArray());
return new ByteArrayInputStream(recv.toByteArray());
}
/*
* Invokes UploadPack with protocol v2 and sends it the given lines.
* Returns UploadPack's output stream, not including the capability
* advertisement by the server.
*/
private ByteArrayInputStream uploadPackV2(RequestPolicy requestPolicy,
RefFilter refFilter, String... inputLines) throws Exception {
ByteArrayInputStream recvStream =
uploadPackV2Setup(requestPolicy, refFilter, inputLines);
PacketLineIn pckIn = new PacketLineIn(recvStream);
// drain capabilities
while (pckIn.readString() != PacketLineIn.END) {
// do nothing
}
return recvStream;
}
private ByteArrayInputStream uploadPackV2(String... inputLines) throws Exception {
return uploadPackV2(null, null, inputLines);
}
@Test
public void testV2Capabilities() throws Exception {
ByteArrayInputStream recvStream =
uploadPackV2Setup(null, null, PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
// capability advertisement (always sent)
assertThat(pckIn.readString(), is("version 2"));
assertThat(
Arrays.asList(pckIn.readString(), pckIn.readString()),
@ -377,13 +405,24 @@ private ByteArrayInputStream uploadPackV2(RequestPolicy requestPolicy,
// allow additional commands to be added to the list,
// and additional capabilities to be added to existing
// commands without requiring test changes.
hasItems("ls-refs", "fetch"));
hasItems("ls-refs", "fetch=shallow"));
assertTrue(pckIn.readString() == PacketLineIn.END);
return recvStream;
}
private ByteArrayInputStream uploadPackV2(String... inputLines) throws Exception {
return uploadPackV2(null, null, inputLines);
@Test
public void testV2CapabilitiesAllowFilter() throws Exception {
server.getConfig().setBoolean("uploadpack", null, "allowfilter", true);
ByteArrayInputStream recvStream =
uploadPackV2Setup(null, null, PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
assertThat(pckIn.readString(), is("version 2"));
assertThat(
Arrays.asList(pckIn.readString(), pckIn.readString()),
// TODO(jonathantanmy) This check overspecifies the
// order of the capabilities of "fetch".
hasItems("ls-refs", "fetch=filter shallow"));
assertTrue(pckIn.readString() == PacketLineIn.END);
}
@Test
@ -514,6 +553,17 @@ public void testV2LsRefsRefPrefixNoSlash() throws Exception {
assertTrue(pckIn.readString() == PacketLineIn.END);
}
@Test
public void testV2LsRefsUnrecognizedArgument() throws Exception {
thrown.expect(PackProtocolException.class);
thrown.expectMessage("unexpected invalid-argument");
uploadPackV2(
"command=ls-refs\n",
PacketLineIn.DELIM,
"invalid-argument\n",
PacketLineIn.END);
}
/*
* Parse multiplexed packfile output from upload-pack using protocol V2
* into the client repository.
@ -877,6 +927,157 @@ public void testV2FetchOfsDelta() throws Exception {
assertTrue(stats.getNumOfsDelta() != 0);
}
@Test
public void testV2FetchShallow() throws Exception {
RevCommit commonParent = remote.commit().message("parent").create();
RevCommit fooChild = remote.commit().message("x").parent(commonParent).create();
RevCommit barChild = remote.commit().message("y").parent(commonParent).create();
remote.update("branch1", barChild);
// Without shallow, the server thinks that we have
// commonParent, so it doesn't send it.
ByteArrayInputStream recvStream = uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + barChild.toObjectId().getName() + "\n",
"have " + fooChild.toObjectId().getName() + "\n",
"done\n",
PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
assertThat(pckIn.readString(), is("packfile"));
parsePack(recvStream);
assertTrue(client.hasObject(barChild.toObjectId()));
assertFalse(client.hasObject(commonParent.toObjectId()));
// With shallow, the server knows that we don't have
// commonParent, so it sends it.
recvStream = uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + barChild.toObjectId().getName() + "\n",
"have " + fooChild.toObjectId().getName() + "\n",
"shallow " + fooChild.toObjectId().getName() + "\n",
"done\n",
PacketLineIn.END);
pckIn = new PacketLineIn(recvStream);
assertThat(pckIn.readString(), is("packfile"));
parsePack(recvStream);
assertTrue(client.hasObject(commonParent.toObjectId()));
}
@Test
public void testV2FetchDeepenAndDone() throws Exception {
RevCommit parent = remote.commit().message("parent").create();
RevCommit child = remote.commit().message("x").parent(parent).create();
remote.update("branch1", child);
// "deepen 1" sends only the child.
ByteArrayInputStream recvStream = uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + child.toObjectId().getName() + "\n",
"deepen 1\n",
"done\n",
PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
assertThat(pckIn.readString(), is("shallow-info"));
assertThat(pckIn.readString(), is("shallow " + child.toObjectId().getName()));
assertThat(pckIn.readString(), theInstance(PacketLineIn.DELIM));
assertThat(pckIn.readString(), is("packfile"));
parsePack(recvStream);
assertTrue(client.hasObject(child.toObjectId()));
assertFalse(client.hasObject(parent.toObjectId()));
// Without that, the parent is sent too.
recvStream = uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + child.toObjectId().getName() + "\n",
"done\n",
PacketLineIn.END);
pckIn = new PacketLineIn(recvStream);
assertThat(pckIn.readString(), is("packfile"));
parsePack(recvStream);
assertTrue(client.hasObject(parent.toObjectId()));
}
@Test
public void testV2FetchDeepenWithoutDone() throws Exception {
RevCommit parent = remote.commit().message("parent").create();
RevCommit child = remote.commit().message("x").parent(parent).create();
remote.update("branch1", child);
ByteArrayInputStream recvStream = uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + child.toObjectId().getName() + "\n",
"deepen 1\n",
PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
// Verify that only the correct section is sent. "shallow-info"
// is not sent because, according to the specification, it is
// sent only if a packfile is sent.
assertThat(pckIn.readString(), is("acknowledgments"));
assertThat(pckIn.readString(), is("NAK"));
assertThat(pckIn.readString(), theInstance(PacketLineIn.END));
}
@Test
public void testV2FetchUnrecognizedArgument() throws Exception {
thrown.expect(PackProtocolException.class);
thrown.expectMessage("unexpected invalid-argument");
uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"invalid-argument\n",
PacketLineIn.END);
}
@Test
public void testV2FetchFilter() throws Exception {
RevBlob big = remote.blob("foobar");
RevBlob small = remote.blob("fooba");
RevTree tree = remote.tree(remote.file("1", big),
remote.file("2", small));
RevCommit commit = remote.commit(tree);
remote.update("master", commit);
server.getConfig().setBoolean("uploadpack", null, "allowfilter", true);
ByteArrayInputStream recvStream = uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + commit.toObjectId().getName() + "\n",
"filter blob:limit=5\n",
"done\n",
PacketLineIn.END);
PacketLineIn pckIn = new PacketLineIn(recvStream);
assertThat(pckIn.readString(), is("packfile"));
parsePack(recvStream);
assertFalse(client.hasObject(big.toObjectId()));
assertTrue(client.hasObject(small.toObjectId()));
}
@Test
public void testV2FetchFilterWhenNotAllowed() throws Exception {
RevCommit commit = remote.commit().message("0").create();
remote.update("master", commit);
server.getConfig().setBoolean("uploadpack", null, "allowfilter", false);
thrown.expect(PackProtocolException.class);
thrown.expectMessage("unexpected filter blob:limit=5");
uploadPackV2(
"command=fetch\n",
PacketLineIn.DELIM,
"want " + commit.toObjectId().getName() + "\n",
"filter blob:limit=5\n",
"done\n",
PacketLineIn.END);
}
private static class RejectAllRefFilter implements RefFilter {
@Override
public Map<String, Ref> filter(Map<String, Ref> refs) {

3
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
View File

@ -224,6 +224,8 @@ credentialPassword=Password
credentialUsername=Username
daemonAlreadyRunning=Daemon already running
daysAgo={0} days ago
deepenNotWithDeepen=Cannot combine deepen with deepen-not
deepenSinceWithDeepen=Cannot combine deepen with deepen-since
deleteBranchUnexpectedResult=Delete branch returned unexpected result {0}
deleteFileFailed=Could not delete file {0}
deleteRequiresZeroNewId=Delete requires new ID to be zero
@ -395,6 +397,7 @@ invalidStageForPath=Invalid stage {0} for path {1}
invalidSystemProperty=Invalid system property ''{0}'': ''{1}''; using default value {2}
invalidTagOption=Invalid tag option: {0}
invalidTimeout=Invalid timeout: {0}
invalidTimestamp=Invalid timestamp in {0}
invalidTimeUnitValue2=Invalid time unit value: {0}.{1}={2}
invalidTimeUnitValue3=Invalid time unit value: {0}.{1}.{2}={3}
invalidTreeZeroLengthName=Cannot append a tree entry with zero-length name

3
org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
View File

@ -285,6 +285,8 @@ public static JGitText get() {
/***/ public String credentialUsername;
/***/ public String daemonAlreadyRunning;
/***/ public String daysAgo;
/***/ public String deepenNotWithDeepen;
/***/ public String deepenSinceWithDeepen;
/***/ public String deleteBranchUnexpectedResult;
/***/ public String deleteFileFailed;
/***/ public String deleteRequiresZeroNewId;
@ -455,6 +457,7 @@ public static JGitText get() {
/***/ public String invalidSystemProperty;
/***/ public String invalidTagOption;
/***/ public String invalidTimeout;
/***/ public String invalidTimestamp;
/***/ public String invalidTimeUnitValue2;
/***/ public String invalidTimeUnitValue3;
/***/ public String invalidTreeZeroLengthName;

8
org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java
View File

@ -107,6 +107,14 @@ public class GitProtocolConstants {
*/
public static final String OPTION_SHALLOW = "shallow"; //$NON-NLS-1$
/**
* The client wants the "deepen" command to be interpreted as relative to
* the client's shallow commits.
*
* @since 5.0
*/
public static final String OPTION_DEEPEN_RELATIVE = "deepen-relative"; //$NON-NLS-1$
/**
* The client does not want progress messages and will ignore them.
*

264
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java
View File

@ -50,6 +50,7 @@
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_DEEPEN_RELATIVE;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_FILTER;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK;
@ -118,14 +119,6 @@
* Implements the server side of a fetch connection, transmitting objects.
*/
public class UploadPack {
// UploadPack sends these lines as the first response to a client that
// supports protocol version 2.
private static final String[] v2CapabilityAdvertisement = {
"version 2", //$NON-NLS-1$
COMMAND_LS_REFS,
COMMAND_FETCH
};
/** Policy the server uses to validate client requests */
public static enum RequestPolicy {
/** Client may only ask for objects the server advertised a reference for. */
@ -299,12 +292,22 @@ public Set<String> getOptions() {
/** Shallow commits the client already has. */
private final Set<ObjectId> clientShallowCommits = new HashSet<>();
/** Shallow commits on the client which are now becoming unshallow */
private final List<ObjectId> unshallowCommits = new ArrayList<>();
/** Desired depth from the client on a shallow request. */
private int depth;
/**
* Commit time of the newest objects the client has asked us using
* --shallow-since not to send. Cannot be nonzero if depth is nonzero.
*/
private int shallowSince;
/**
* (Possibly short) ref names, ancestors of which the client has asked us
* not to send using --shallow-exclude. Cannot be non-null if depth is
* nonzero.
*/
private @Nullable List<String> shallowExcludeRefs;
/** Commit time of the oldest common commit, in seconds. */
private int oldestTime;
@ -786,6 +789,7 @@ private void service() throws IOException {
// If it's a non-bidi request, we need to read the entire request before
// writing a response. Buffer the response until then.
PackStatistics.Accumulator accumulator = new PackStatistics.Accumulator();
List<ObjectId> unshallowCommits = new ArrayList<>();
try {
if (biDirectionalPipe)
sendAdvertisedRefs(new PacketLineOutRefAdvertiser(pckOut));
@ -815,7 +819,7 @@ else if (requestValidator instanceof AnyRequestValidator)
if (!clientShallowCommits.isEmpty())
verifyClientShallow();
if (depth != 0)
processShallow();
processShallow(null, unshallowCommits, true);
if (!clientShallowCommits.isEmpty())
walk.assumeShallow(clientShallowCommits);
sendPack = negotiate(accumulator);
@ -867,8 +871,9 @@ else if (requestValidator instanceof AnyRequestValidator)
rawOut.stopBuffering();
}
if (sendPack)
sendPack(accumulator, refs == null ? null : refs.values());
if (sendPack) {
sendPack(accumulator, refs == null ? null : refs.values(), unshallowCommits);
}
}
private void lsRefsV2() throws IOException {
@ -953,6 +958,7 @@ private void fetchV2() throws IOException {
}
boolean includeTag = false;
boolean filterReceived = false;
while ((line = pckIn.readString()) != PacketLineIn.END) {
if (line.startsWith("want ")) { //$NON-NLS-1$
wantIds.add(ObjectId.fromString(line.substring(5)));
@ -969,12 +975,74 @@ private void fetchV2() throws IOException {
includeTag = true;
} else if (line.equals(OPTION_OFS_DELTA)) {
options.add(OPTION_OFS_DELTA);
} else if (line.startsWith("shallow ")) { //$NON-NLS-1$
clientShallowCommits.add(ObjectId.fromString(line.substring(8)));
} else if (line.startsWith("deepen ")) { //$NON-NLS-1$
depth = Integer.parseInt(line.substring(7));
if (depth <= 0) {
throw new PackProtocolException(
MessageFormat.format(JGitText.get().invalidDepth,
Integer.valueOf(depth)));
}
if (shallowSince != 0) {
throw new PackProtocolException(
JGitText.get().deepenSinceWithDeepen);
}
if (shallowExcludeRefs != null) {
throw new PackProtocolException(
JGitText.get().deepenNotWithDeepen);
}
} else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$
List<String> exclude = shallowExcludeRefs;
if (exclude == null) {
exclude = shallowExcludeRefs = new ArrayList<>();
}
exclude.add(line.substring(11));
if (depth != 0) {
throw new PackProtocolException(
JGitText.get().deepenNotWithDeepen);
}
} else if (line.equals(OPTION_DEEPEN_RELATIVE)) {
options.add(OPTION_DEEPEN_RELATIVE);
} else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$
shallowSince = Integer.parseInt(line.substring(13));
if (shallowSince <= 0) {
throw new PackProtocolException(
MessageFormat.format(
JGitText.get().invalidTimestamp, line));
}
if (depth != 0) {
throw new PackProtocolException(
JGitText.get().deepenSinceWithDeepen);
}
} else if (transferConfig.isAllowFilter()
&& line.startsWith(OPTION_FILTER + ' ')) {
if (filterReceived) {
throw new PackProtocolException(JGitText.get().tooManyFilters);
}
filterReceived = true;
parseFilter(line.substring(OPTION_FILTER.length() + 1));
} else {
throw new PackProtocolException(MessageFormat
.format(JGitText.get().unexpectedPacketLine, line));
}
// else ignore it
}
rawOut.stopBuffering();
boolean sectionSent = false;
@Nullable List<ObjectId> shallowCommits = null;
List<ObjectId> unshallowCommits = new ArrayList<>();
if (!clientShallowCommits.isEmpty()) {
verifyClientShallow();
}
if (depth != 0 || shallowSince != 0 || shallowExcludeRefs != null) {
shallowCommits = new ArrayList<ObjectId>();
processShallow(shallowCommits, unshallowCommits, false);
}
if (!clientShallowCommits.isEmpty())
walk.assumeShallow(clientShallowCommits);
if (doneReceived) {
processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE));
} else {
@ -992,14 +1060,29 @@ private void fetchV2() throws IOException {
}
sectionSent = true;
}
if (doneReceived || okToGiveUp()) {
if (shallowCommits != null) {
if (sectionSent)
pckOut.writeDelim();
pckOut.writeString("shallow-info\n"); //$NON-NLS-1$
for (ObjectId o : shallowCommits) {
pckOut.writeString("shallow " + o.getName() + '\n'); //$NON-NLS-1$
}
for (ObjectId o : unshallowCommits) {
pckOut.writeString("unshallow " + o.getName() + '\n'); //$NON-NLS-1$
}
sectionSent = true;
}
if (sectionSent)
pckOut.writeDelim();
pckOut.writeString("packfile\n"); //$NON-NLS-1$
sendPack(new PackStatistics.Accumulator(),
includeTag
? db.getRefDatabase().getRefsByPrefix(R_TAGS)
: null);
: null,
new ArrayList<ObjectId>());
}
pckOut.end();
}
@ -1034,13 +1117,24 @@ private boolean serveOneCommandV2() throws IOException {
.format(JGitText.get().unknownTransportCommand, command));
}
private List<String> getV2CapabilityAdvertisement() {
ArrayList<String> caps = new ArrayList<>();
caps.add("version 2"); //$NON-NLS-1$
caps.add(COMMAND_LS_REFS);
caps.add(
COMMAND_FETCH + '=' +
(transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$
OPTION_SHALLOW);
return caps;
}
private void serviceV2() throws IOException {
if (biDirectionalPipe) {
// Just like in service(), the capability advertisement
// is sent only if this is a bidirectional pipe. (If
// not, the client is expected to call
// sendAdvertisedRefs() on its own.)
for (String s : v2CapabilityAdvertisement) {
for (String s : getV2CapabilityAdvertisement()) {
pckOut.writeString(s + "\n"); //$NON-NLS-1$
}
pckOut.end();
@ -1076,7 +1170,23 @@ private static Set<ObjectId> refIdSet(Collection<Ref> refs) {
return ids;
}
private void processShallow() throws IOException {
/*
* Determines what "shallow" and "unshallow" lines to send to the user.
* The information is written to shallowCommits (if not null) and
* unshallowCommits, and also written to #pckOut (if writeToPckOut is
* true).
*/
private void processShallow(@Nullable List<ObjectId> shallowCommits,
List<ObjectId> unshallowCommits,
boolean writeToPckOut) throws IOException {
if (options.contains(OPTION_DEEPEN_RELATIVE) ||
shallowSince != 0 ||
shallowExcludeRefs != null) {
// TODO(jonathantanmy): Implement deepen-relative, deepen-since,
// and deepen-not.
throw new UnsupportedOperationException();
}
int walkDepth = depth - 1;
try (DepthWalk.RevWalk depthWalk = new DepthWalk.RevWalk(
walk.getObjectReader(), walkDepth)) {
@ -1097,19 +1207,29 @@ private void processShallow() throws IOException {
// Commits at the boundary which aren't already shallow in
// the client need to be marked as such
if (c.getDepth() == walkDepth
&& !clientShallowCommits.contains(c))
pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
&& !clientShallowCommits.contains(c)) {
if (shallowCommits != null) {
shallowCommits.add(c.copy());
}
if (writeToPckOut) {
pckOut.writeString("shallow " + o.name()); //$NON-NLS-1$
}
}
// Commits not on the boundary which are shallow in the client
// need to become unshallowed
if (c.getDepth() < walkDepth
&& clientShallowCommits.remove(c)) {
unshallowCommits.add(c.copy());
pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
if (writeToPckOut) {
pckOut.writeString("unshallow " + c.name()); //$NON-NLS-1$
}
}
}
}
pckOut.end();
if (writeToPckOut) {
pckOut.end();
}
}
private void verifyClientShallow()
@ -1147,16 +1267,38 @@ private void verifyClientShallow()
* @param adv
* the advertisement formatter.
* @throws java.io.IOException
* the formatter failed to write an advertisement.
* the formatter failed to write an advertisement.
* @throws org.eclipse.jgit.transport.ServiceMayNotContinueException
* the hook denied advertisement.
* the hook denied advertisement.
*/
public void sendAdvertisedRefs(RefAdvertiser adv) throws IOException,
ServiceMayNotContinueException {
sendAdvertisedRefs(adv, null);
}
/**
* Generate an advertisement of available refs and capabilities.
*
* @param adv
* the advertisement formatter.
* @param serviceName
* if not null, also output "# service=serviceName" followed by a
* flush packet before the advertisement. This is required
* in v0 of the HTTP protocol, described in Git's
* Documentation/technical/http-protocol.txt.
* @throws java.io.IOException
* the formatter failed to write an advertisement.
* @throws org.eclipse.jgit.transport.ServiceMayNotContinueException
* the hook denied advertisement.
* @since 5.0
*/
public void sendAdvertisedRefs(RefAdvertiser adv,
@Nullable String serviceName) throws IOException,
ServiceMayNotContinueException {
if (useProtocolV2()) {
// The equivalent in v2 is only the capabilities
// advertisement.
for (String s : v2CapabilityAdvertisement) {
for (String s : getV2CapabilityAdvertisement()) {
adv.writeOne(s);
}
adv.end();
@ -1173,6 +1315,10 @@ public void sendAdvertisedRefs(RefAdvertiser adv) throws IOException,
throw fail;
}
if (serviceName != null) {
adv.writeOne("# service=" + serviceName + '\n'); //$NON-NLS-1$
adv.end();
}
adv.init(db);
adv.advertiseCapability(OPTION_INCLUDE_TAG);
adv.advertiseCapability(OPTION_MULTI_ACK_DETAILED);
@ -1236,6 +1382,33 @@ public OutputStream getMessageOutputStream() {
return msgOut;
}
private void parseFilter(String arg) throws PackProtocolException {
if (arg.equals("blob:none")) { //$NON-NLS-1$
filterBlobLimit = 0;
} else if (arg.startsWith("blob:limit=")) { //$NON-NLS-1$
try {
filterBlobLimit = Long.parseLong(
arg.substring("blob:limit=".length())); //$NON-NLS-1$
} catch (NumberFormatException e) {
throw new PackProtocolException(
MessageFormat.format(JGitText.get().invalidFilter,
arg));
}
}
/*
* We must have (1) either "blob:none" or
* "blob:limit=" set (because we only support
* blob size limits for now), and (2) if the
* latter, then it must be nonnegative. Throw
* if (1) or (2) is not met.
*/
if (filterBlobLimit < 0) {
throw new PackProtocolException(
MessageFormat.format(JGitText.get().invalidFilter,
arg));
}
}
private void recvWants() throws IOException {
boolean isFirst = true;
boolean filterReceived = false;
@ -1276,30 +1449,7 @@ private void recvWants() throws IOException {
}
filterReceived = true;
if (arg.equals("blob:none")) { //$NON-NLS-1$
filterBlobLimit = 0;
} else if (arg.startsWith("blob:limit=")) { //$NON-NLS-1$
try {
filterBlobLimit = Long.parseLong(
arg.substring("blob:limit=".length())); //$NON-NLS-1$
} catch (NumberFormatException e) {
throw new PackProtocolException(
MessageFormat.format(JGitText.get().invalidFilter,
arg));
}
}
/*
* We must have (1) either "blob:none" or
* "blob:limit=" set (because we only support
* blob size limits for now), and (2) if the
* latter, then it must be nonnegative. Throw
* if (1) or (2) is not met.
*/
if (filterBlobLimit < 0) {
throw new PackProtocolException(
MessageFormat.format(JGitText.get().invalidFilter,
arg));
}
parseFilter(arg);
continue;
}
@ -1754,16 +1904,20 @@ private boolean wantSatisfied(RevObject want) throws IOException {
* refs to search for annotated tags to include in the pack
* if the {@link #OPTION_INCLUDE_TAG} capability was
* requested.
* @param unshallowCommits
* shallow commits on the client that are now becoming
* unshallow
* @throws IOException
* if an error occured while generating or writing the pack.
*/
private void sendPack(PackStatistics.Accumulator accumulator,
@Nullable Collection<Ref> allTags) throws IOException {
@Nullable Collection<Ref> allTags,
List<ObjectId> unshallowCommits) throws IOException {
final boolean sideband = options.contains(OPTION_SIDE_BAND)
|| options.contains(OPTION_SIDE_BAND_64K);
if (sideband) {
try {
sendPack(true, accumulator, allTags);
sendPack(true, accumulator, allTags, unshallowCommits);
} catch (ServiceMayNotContinueException noPack) {
// This was already reported on (below).
throw noPack;
@ -1784,7 +1938,7 @@ private void sendPack(PackStatistics.Accumulator accumulator,
throw err;
}
} else {
sendPack(false, accumulator, allTags);
sendPack(false, accumulator, allTags, unshallowCommits);
}
}
@ -1816,12 +1970,16 @@ private boolean reportInternalServerErrorOverSideband() {
* refs to search for annotated tags to include in the pack
* if the {@link #OPTION_INCLUDE_TAG} capability was
* requested.
* @param unshallowCommits
* shallow commits on the client that are now becoming
* unshallow
* @throws IOException
* if an error occured while generating or writing the pack.
*/
private void sendPack(final boolean sideband,
PackStatistics.Accumulator accumulator,
@Nullable Collection<Ref> allTags) throws IOException {
@Nullable Collection<Ref> allTags,
List<ObjectId> unshallowCommits) throws IOException {
ProgressMonitor pm = NullProgressMonitor.INSTANCE;
OutputStream packOut = rawOut;