From f802f06e7fd5a98f256b7b7727598491f563bf2f Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sun, 2 Aug 2020 19:22:05 +0200 Subject: [PATCH] Client-side protocol V2 support for fetching Make all transports request protocol V2 when fetching. Depending on the transport, set the GIT_PROTOCOL environment variable (file and ssh), pass the Git-Protocol header (http), or set the hidden "\0version=2\0" (git anon). We'll fall back to V0 if the server doesn't reply with a version 2 answer. A user can control which protocol the client requests via the git config protocol.version; if not set, JGit requests protocol V2 for fetching. Pushing always uses protocol V0 still. In the API, there is only a new Transport.openFetch() version that takes a collection of RefSpecs plus additional patterns to construct the Ref prefixes for the "ls-refs" command in protocol V2. If none are given, the server will still advertise all refs, even in protocol V2. BasePackConnection.readAdvertisedRefs() handles falling back to protocol V0. It newly returns true if V0 was used and the advertised refs were read, and false if V2 is used and an explicit "ls-refs" is needed. (This can't be done transparently inside readAdvertisedRefs() because a "stateless RPC" transport like TransportHttp may need to open a new connection for writing.) BasePackFetchConnection implements the changes needed for the protocol V2 "fetch" command (simplified ACK handling, delimiters, section headers). In TransportHttp, change readSmartHeaders() to also recognize the "version 2" packet line as a valid smart server indication. Adapt tests, and run all the HTTP tests not only with both HTTP connection factories (JDK and Apache HttpClient) but also with both protocol V0 and V2. Do the same for the SSH transport tests. Bug: 553083 Change-Id: Ice9866aa78020f5ca8f397cde84dc224bf5d41b4 Signed-off-by: Thomas Wolf Signed-off-by: Matthias Sohn --- .../http/test/AllProtocolsHttpTestCase.java | 99 +++++ .../http/test/DumbClientDumbServerTest.java | 10 +- .../http/test/DumbClientSmartServerTest.java | 10 +- .../test/SmartClientSmartServerSslTest.java | 17 +- .../http/test/SmartClientSmartServerTest.java | 249 ++++++++---- .../sshd/ApacheSshProtocol2Test.java | 24 ++ .../jgit/transport/sshd/SshdSession.java | 17 +- .../jgit/transport/JSchSshProtocol2Test.java | 26 ++ .../eclipse/jgit/transport/JschSession.java | 25 +- .../eclipse/jgit/internal/JGitText.properties | 1 + .../org/eclipse/jgit/api/LsRemoteCommand.java | 2 +- .../org/eclipse/jgit/internal/JGitText.java | 1 + .../jgit/transport/BasePackConnection.java | 363 +++++++++++++++--- .../transport/BasePackFetchConnection.java | 187 ++++++++- .../eclipse/jgit/transport/FetchProcess.java | 29 +- .../jgit/transport/GitProtocolConstants.java | 68 ++++ .../eclipse/jgit/transport/PacketLineIn.java | 20 +- .../eclipse/jgit/transport/RefAdvertiser.java | 8 +- .../jgit/transport/RemoteSession2.java | 45 +++ .../org/eclipse/jgit/transport/Transport.java | 43 +++ .../jgit/transport/TransportGitAnon.java | 33 +- .../jgit/transport/TransportGitSsh.java | 65 +++- .../eclipse/jgit/transport/TransportHttp.java | 117 ++++-- .../jgit/transport/TransportLocal.java | 50 ++- .../eclipse/jgit/transport/UploadPack.java | 12 +- 25 files changed, 1301 insertions(+), 220 deletions(-) create mode 100644 org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java create mode 100644 org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java create mode 100644 org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java new file mode 100644 index 000000000..c6931ad5b --- /dev/null +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2020, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.http.test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.eclipse.jgit.junit.http.HttpTestCase; +import org.eclipse.jgit.transport.HttpTransport; +import org.eclipse.jgit.transport.http.HttpConnectionFactory; +import org.eclipse.jgit.transport.http.JDKHttpConnectionFactory; +import org.eclipse.jgit.transport.http.apache.HttpClientConnectionFactory; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Abstract test base class for running HTTP-related tests with all connection + * factories provided in JGit and with both protocol V0 and V2. + */ +@Ignore +@RunWith(Parameterized.class) +public abstract class AllProtocolsHttpTestCase extends HttpTestCase { + + protected static class TestParameters { + + public final HttpConnectionFactory factory; + + public final boolean enableProtocolV2; + + public TestParameters(HttpConnectionFactory factory, + boolean enableProtocolV2) { + this.factory = factory; + this.enableProtocolV2 = enableProtocolV2; + } + + @Override + public String toString() { + return factory.toString() + " protocol " + + (enableProtocolV2 ? "V2" : "V0"); + } + } + + @Parameters(name = "{0}") + public static Collection data() { + // run all tests with both connection factories we have + HttpConnectionFactory[] factories = new HttpConnectionFactory[] { + new JDKHttpConnectionFactory() { + + @Override + public String toString() { + return this.getClass().getSuperclass().getName(); + } + }, new HttpClientConnectionFactory() { + + @Override + public String toString() { + return this.getClass().getSuperclass().getName(); + } + } }; + List result = new ArrayList<>(); + for (HttpConnectionFactory factory : factories) { + result.add(new TestParameters(factory, false)); + result.add(new TestParameters(factory, true)); + } + return result; + } + + protected final boolean enableProtocolV2; + + protected AllProtocolsHttpTestCase(TestParameters params) { + HttpTransport.setConnectionFactory(params.factory); + enableProtocolV2 = params.enableProtocolV2; + } + + private static HttpConnectionFactory originalFactory; + + @BeforeClass + public static void saveConnectionFactory() { + originalFactory = HttpTransport.getConnectionFactory(); + } + + @AfterClass + public static void restoreConnectionFactory() { + HttpTransport.setConnectionFactory(originalFactory); + } + +} diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java index 6da5f86b3..ef9c41e62 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java @@ -42,11 +42,10 @@ import org.eclipse.jgit.transport.Transport; import org.eclipse.jgit.transport.TransportHttp; import org.eclipse.jgit.transport.URIish; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.junit.Before; import org.junit.Test; -public class DumbClientDumbServerTest extends AllFactoriesHttpTestCase { +public class DumbClientDumbServerTest extends AllProtocolsHttpTestCase { private Repository remoteRepository; private URIish remoteURI; @@ -55,8 +54,8 @@ public class DumbClientDumbServerTest extends AllFactoriesHttpTestCase { private RevCommit A, B; - public DumbClientDumbServerTest(HttpConnectionFactory cf) { - super(cf); + public DumbClientDumbServerTest(TestParameters params) { + super(params); } @Override @@ -77,6 +76,9 @@ public void setUp() throws Exception { remoteRepository = src.getRepository(); remoteURI = toURIish(app, srcGit.getName()); + if (enableProtocolV2) { + remoteRepository.getConfig().setInt("protocol", null, "version", 2); + } A_txt = src.blob("A"); A = src.commit().add("A_txt", A_txt).create(); diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java index ccde1fe55..096a199d0 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java @@ -42,11 +42,10 @@ import org.eclipse.jgit.transport.Transport; import org.eclipse.jgit.transport.TransportHttp; import org.eclipse.jgit.transport.URIish; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.junit.Before; import org.junit.Test; -public class DumbClientSmartServerTest extends AllFactoriesHttpTestCase { +public class DumbClientSmartServerTest extends AllProtocolsHttpTestCase { private Repository remoteRepository; private URIish remoteURI; @@ -55,8 +54,8 @@ public class DumbClientSmartServerTest extends AllFactoriesHttpTestCase { private RevCommit A, B; - public DumbClientSmartServerTest(HttpConnectionFactory cf) { - super(cf); + public DumbClientSmartServerTest(TestParameters params) { + super(params); } @Override @@ -76,6 +75,9 @@ public void setUp() throws Exception { remoteRepository = src.getRepository(); remoteURI = toURIish(app, srcName); + if (enableProtocolV2) { + remoteRepository.getConfig().setInt("protocol", null, "version", 2); + } A_txt = src.blob("A"); A = src.commit().add("A_txt", A_txt).create(); diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java index 597fb2e50..3a43c6f4c 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Thomas Wolf and others + * Copyright (C) 2017, 2020 Thomas Wolf and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -48,7 +48,6 @@ import org.eclipse.jgit.transport.Transport; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.eclipse.jgit.util.HttpSupport; import org.junit.Before; import org.junit.Test; @@ -56,7 +55,7 @@ import org.junit.runners.Parameterized; @RunWith(Parameterized.class) -public class SmartClientSmartServerSslTest extends AllFactoriesHttpTestCase { +public class SmartClientSmartServerSslTest extends AllProtocolsHttpTestCase { // We run these tests with a server on localhost with a self-signed // certificate. We don't do authentication tests here, so there's no need @@ -112,8 +111,8 @@ public boolean get(URIish uri, CredentialItem... items) private RevCommit A, B; - public SmartClientSmartServerSslTest(HttpConnectionFactory cf) { - super(cf); + public SmartClientSmartServerSslTest(TestParameters params) { + super(params); } @Override @@ -132,6 +131,10 @@ public void setUp() throws Exception { .getConfig() .setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, true); + if (enableProtocolV2) { + src.getRepository().getConfig().setInt("protocol", null, "version", + 2); + } GitServlet gs = new GitServlet(); @@ -238,7 +241,7 @@ public void testInitialClone_ViaHttps() throws Exception { fsck(dst, B); List requests = getRequests(); - assertEquals(2, requests.size()); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); } @Test @@ -256,7 +259,7 @@ public void testInitialClone_RedirectToHttps() throws Exception { fsck(dst, B); List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); } @Test diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java index 8d1870a87..9e88cefc6 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2017 Google Inc. and others + * Copyright (C) 2010, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -86,13 +86,12 @@ import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UploadPack; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.eclipse.jgit.util.HttpSupport; import org.eclipse.jgit.util.SystemReader; import org.junit.Before; import org.junit.Test; -public class SmartClientSmartServerTest extends AllFactoriesHttpTestCase { +public class SmartClientSmartServerTest extends AllProtocolsHttpTestCase { private static final String HDR_TRANSFER_ENCODING = "Transfer-Encoding"; private AdvertiseRefsHook advertiseRefsHook; @@ -120,8 +119,8 @@ public class SmartClientSmartServerTest extends AllFactoriesHttpTestCase { private RevCommit A, B, unreachableCommit; - public SmartClientSmartServerTest(HttpConnectionFactory cf) { - super(cf); + public SmartClientSmartServerTest(TestParameters params) { + super(params); } @Override @@ -132,9 +131,12 @@ public void setUp() throws Exception { final TestRepository src = createTestRepository(); final String srcName = src.getRepository().getDirectory().getName(); src.getRepository() - .getConfig() - .setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, true); + .getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, + null, ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, + true); + if (enableProtocolV2) { + src.getRepository().getConfig().setInt("protocol", null, "version", 2); + } GitServlet gs = new GitServlet(); gs.setUploadPackFactory((HttpServletRequest req, Repository db) -> { @@ -448,7 +450,7 @@ public void testListRemote() throws IOException { assertEquals(B, map.get(Constants.HEAD).getObjectId()); List requests = getRequests(); - assertEquals(1, requests.size()); + assertEquals(enableProtocolV2 ? 2 : 1, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -458,7 +460,22 @@ public void testListRemote() throws IOException { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info .getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(1); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -576,9 +593,10 @@ public void testInitialClone_Small() throws Exception { } List requests = getRequests(); - assertEquals(2, requests.size()); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); - AccessEvent info = requests.get(0); + int requestNumber = 0; + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(remoteURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -586,9 +604,24 @@ public void testInitialClone_Small() throws Exception { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info .getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } - AccessEvent service = requests.get(1); + AccessEvent service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -628,7 +661,8 @@ private void initialClone_Redirect(int nofRedirects, int code, } List requests = getRequests(); - assertEquals(2 + nofRedirects, requests.size()); + assertEquals((enableProtocolV2 ? 3 : 2) + nofRedirects, + requests.size()); int n = 0; while (n < nofRedirects) { @@ -644,7 +678,22 @@ private void initialClone_Redirect(int nofRedirects, int code, assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(n++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } AccessEvent service = requests.get(n++); assertEquals("POST", service.getMethod()); @@ -756,7 +805,7 @@ public void testInitialClone_RedirectOnPostAllowed() throws Exception { } List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -766,24 +815,27 @@ public void testInitialClone_RedirectOnPostAllowed() throws Exception { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } AccessEvent redirect = requests.get(1); assertEquals("POST", redirect.getMethod()); assertEquals(301, redirect.getStatus()); - AccessEvent service = requests.get(2); - assertEquals("POST", service.getMethod()); - assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); - assertEquals(0, service.getParameters().size()); - assertNotNull("has content-length", - service.getRequestHeader(HDR_CONTENT_LENGTH)); - assertNull("not chunked", - service.getRequestHeader(HDR_TRANSFER_ENCODING)); - - assertEquals(200, service.getStatus()); - assertEquals("application/x-git-upload-pack-result", - service.getResponseHeader(HDR_CONTENT_TYPE)); + for (int i = 2; i < requests.size(); i++) { + AccessEvent service = requests.get(i); + assertEquals("POST", service.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); + assertEquals(0, service.getParameters().size()); + assertNotNull("has content-length", + service.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + service.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals(200, service.getStatus()); + assertEquals("application/x-git-upload-pack-result", + service.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -830,7 +882,7 @@ public void testInitialClone_WithAuthentication() throws Exception { } List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -844,20 +896,24 @@ public void testInitialClone_WithAuthentication() throws Exception { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } - AccessEvent service = requests.get(2); - assertEquals("POST", service.getMethod()); - assertEquals(join(authURI, "git-upload-pack"), service.getPath()); - assertEquals(0, service.getParameters().size()); - assertNotNull("has content-length", - service.getRequestHeader(HDR_CONTENT_LENGTH)); - assertNull("not chunked", - service.getRequestHeader(HDR_TRANSFER_ENCODING)); + for (int i = 2; i < requests.size(); i++) { + AccessEvent service = requests.get(i); + assertEquals("POST", service.getMethod()); + assertEquals(join(authURI, "git-upload-pack"), service.getPath()); + assertEquals(0, service.getParameters().size()); + assertNotNull("has content-length", + service.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + service.getRequestHeader(HDR_TRANSFER_ENCODING)); - assertEquals(200, service.getStatus()); - assertEquals("application/x-git-upload-pack-result", - service.getResponseHeader(HDR_CONTENT_TYPE)); + assertEquals(200, service.getStatus()); + assertEquals("application/x-git-upload-pack-result", + service.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -937,19 +993,20 @@ public boolean get(URIish uri, CredentialItem... items) } List requests = getRequests(); - assertEquals(4, requests.size()); + assertEquals(enableProtocolV2 ? 5 : 4, requests.size()); - AccessEvent redirect = requests.get(0); + int requestNumber = 0; + AccessEvent redirect = requests.get(requestNumber++); assertEquals("GET", redirect.getMethod()); assertEquals(join(cloneFrom, "info/refs"), redirect.getPath()); assertEquals(301, redirect.getStatus()); - AccessEvent info = requests.get(1); + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(authURI, "info/refs"), info.getPath()); assertEquals(401, info.getStatus()); - info = requests.get(2); + info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(authURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -957,9 +1014,24 @@ public boolean get(URIish uri, CredentialItem... items) assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(authURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } - AccessEvent service = requests.get(3); + AccessEvent service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(authURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -987,7 +1059,7 @@ public void testInitialClone_WithAuthenticationOnPostOnly() } List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -997,25 +1069,30 @@ public void testInitialClone_WithAuthenticationOnPostOnly() assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } AccessEvent service = requests.get(1); assertEquals("POST", service.getMethod()); assertEquals(join(authOnPostURI, "git-upload-pack"), service.getPath()); assertEquals(401, service.getStatus()); - service = requests.get(2); - assertEquals("POST", service.getMethod()); - assertEquals(join(authOnPostURI, "git-upload-pack"), service.getPath()); - assertEquals(0, service.getParameters().size()); - assertNotNull("has content-length", - service.getRequestHeader(HDR_CONTENT_LENGTH)); - assertNull("not chunked", - service.getRequestHeader(HDR_TRANSFER_ENCODING)); + for (int i = 2; i < requests.size(); i++) { + service = requests.get(i); + assertEquals("POST", service.getMethod()); + assertEquals(join(authOnPostURI, "git-upload-pack"), + service.getPath()); + assertEquals(0, service.getParameters().size()); + assertNotNull("has content-length", + service.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + service.getRequestHeader(HDR_TRANSFER_ENCODING)); - assertEquals(200, service.getStatus()); - assertEquals("application/x-git-upload-pack-result", - service.getResponseHeader(HDR_CONTENT_TYPE)); + assertEquals(200, service.getStatus()); + assertEquals("application/x-git-upload-pack-result", + service.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -1052,9 +1129,11 @@ public void testFetch_FewLocalCommits() throws Exception { List requests = getRequests(); requests.removeAll(cloneRequests); - assertEquals(2, requests.size()); - AccessEvent info = requests.get(0); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); + + int requestNumber = 0; + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(remoteURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -1063,9 +1142,24 @@ public void testFetch_FewLocalCommits() throws Exception { assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); + if (enableProtocolV2) { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } + // We should have needed one request to perform the fetch. // - AccessEvent service = requests.get(1); + AccessEvent service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -1116,9 +1210,10 @@ public void testFetch_TooManyLocalCommits() throws Exception { List requests = getRequests(); requests.removeAll(cloneRequests); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); - AccessEvent info = requests.get(0); + int requestNumber = 0; + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(remoteURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -1127,10 +1222,25 @@ public void testFetch_TooManyLocalCommits() throws Exception { assertEquals("application/x-git-upload-pack-advertisement", info .getResponseHeader(HDR_CONTENT_TYPE)); + if (enableProtocolV2) { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } + // We should have needed two requests to perform the fetch // due to the high number of local unknown commits. // - AccessEvent service = requests.get(1); + AccessEvent service = requests.get(requestNumber++); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -1143,7 +1253,7 @@ public void testFetch_TooManyLocalCommits() throws Exception { assertEquals("application/x-git-upload-pack-result", service .getResponseHeader(HDR_CONTENT_TYPE)); - service = requests.get(2); + service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -1429,5 +1539,4 @@ private void enableReceivePack() throws IOException { cfg.setBoolean("http", null, "receivepack", true); cfg.save(); } - } diff --git a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java new file mode 100644 index 000000000..6bad39e74 --- /dev/null +++ b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport.sshd; + +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; + +public class ApacheSshProtocol2Test extends ApacheSshTest { + + @Override + public void setUp() throws Exception { + super.setUp(); + StoredConfig config = ((Repository) db).getConfig(); + config.setInt("protocol", null, "version", 2); + config.save(); + } +} diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java index 0fb0610b9..5a50cc8f2 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java @@ -24,6 +24,7 @@ import java.util.EnumSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -53,7 +54,7 @@ import org.eclipse.jgit.internal.transport.sshd.JGitSshClient; import org.eclipse.jgit.internal.transport.sshd.SshdText; import org.eclipse.jgit.transport.FtpChannel; -import org.eclipse.jgit.transport.RemoteSession; +import org.eclipse.jgit.transport.RemoteSession2; import org.eclipse.jgit.transport.SshConstants; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.util.StringUtils; @@ -61,11 +62,12 @@ import org.slf4j.LoggerFactory; /** - * An implementation of {@link RemoteSession} based on Apache MINA sshd. + * An implementation of {@link org.eclipse.jgit.transport.RemoteSession + * RemoteSession} based on Apache MINA sshd. * * @since 5.2 */ -public class SshdSession implements RemoteSession { +public class SshdSession implements RemoteSession2 { private static final Logger LOG = LoggerFactory .getLogger(SshdSession.class); @@ -290,8 +292,15 @@ private void notifyCloseListeners() { @Override public Process exec(String commandName, int timeout) throws IOException { + return exec(commandName, Collections.emptyMap(), timeout); + } + + @Override + public Process exec(String commandName, Map environment, + int timeout) throws IOException { @SuppressWarnings("resource") - ChannelExec exec = session.createExecChannel(commandName); + ChannelExec exec = session.createExecChannel(commandName, null, + environment); if (timeout <= 0) { try { exec.open().verify(); diff --git a/org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java b/org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java new file mode 100644 index 000000000..f094448e5 --- /dev/null +++ b/org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +//TODO(ms): move to org.eclipse.jgit.ssh.jsch in 6.0 +package org.eclipse.jgit.transport; + +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; + +public class JSchSshProtocol2Test extends JSchSshTest { + + @Override + public void setUp() throws Exception { + super.setUp(); + StoredConfig config = ((Repository) db).getConfig(); + config.setInt("protocol", null, "version", 2); + config.save(); + } +} diff --git a/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java b/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java index 858bdf3f7..c7d0941b6 100644 --- a/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java +++ b/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java @@ -22,7 +22,9 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -44,7 +46,7 @@ * {@link org.eclipse.jgit.transport.JschConfigSessionFactory} is used to create * the actual session passed to the constructor. */ -public class JschSession implements RemoteSession { +public class JschSession implements RemoteSession2 { final Session sock; final URIish uri; @@ -65,7 +67,14 @@ public JschSession(Session session, URIish uri) { /** {@inheritDoc} */ @Override public Process exec(String command, int timeout) throws IOException { - return new JschProcess(command, timeout); + return exec(command, Collections.emptyMap(), timeout); + } + + /** {@inheritDoc} */ + @Override + public Process exec(String command, Map environment, + int timeout) throws IOException { + return new JschProcess(command, environment, timeout); } /** {@inheritDoc} */ @@ -124,6 +133,8 @@ private class JschProcess extends Process { * * @param commandName * the command to execute + * @param environment + * environment variables to pass on * @param tms * the timeout value, in seconds, for the command. * @throws TransportException @@ -132,11 +143,17 @@ private class JschProcess extends Process { * @throws IOException * on problems opening streams */ - JschProcess(String commandName, int tms) - throws TransportException, IOException { + JschProcess(String commandName, Map environment, + int tms) throws TransportException, IOException { timeout = tms; try { channel = (ChannelExec) sock.openChannel("exec"); //$NON-NLS-1$ + if (environment != null) { + for (Map.Entry envVar : environment + .entrySet()) { + channel.setEnv(envVar.getKey(), envVar.getValue()); + } + } channel.setCommand(commandName); setupStreams(); channel.connect(timeout > 0 ? timeout * 1000 : 0); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index c9e48a5c0..3027e39a7 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -232,6 +232,7 @@ downloadCancelled=Download cancelled downloadCancelledDuringIndexing=Download cancelled during indexing duplicateAdvertisementsOf=duplicate advertisements of {0} duplicateRef=Duplicate ref: {0} +duplicateRefAttribute=Duplicate ref attribute: {0} duplicateRemoteRefUpdateIsIllegal=Duplicate remote ref update is illegal. Affected remote name: {0} duplicateStagesNotAllowed=Duplicate stages not allowed eitherGitDirOrWorkTreeRequired=One of setGitDir or setWorkTree must be called. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java index a4ca30909..77ecf3b36 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java @@ -164,7 +164,7 @@ private Map execute() throws GitAPIException, refSpecs.add(new RefSpec("refs/heads/*:refs/remotes/origin/*")); //$NON-NLS-1$ Collection refs; Map refmap = new HashMap<>(); - try (FetchConnection fc = transport.openFetch()) { + try (FetchConnection fc = transport.openFetch(refSpecs)) { refs = fc.getRefs(); if (refSpecs.isEmpty()) for (Ref r : refs) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index b761210e6..9887ed823 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -260,6 +260,7 @@ public static JGitText get() { /***/ public String downloadCancelledDuringIndexing; /***/ public String duplicateAdvertisementsOf; /***/ public String duplicateRef; + /***/ public String duplicateRefAttribute; /***/ public String duplicateRemoteRefUpdateIsIllegal; /***/ public String duplicateStagesNotAllowed; /***/ public String eitherGitDirOrWorkTreeRequired; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java index 1417faee8..c9c6c0c25 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java @@ -13,7 +13,12 @@ package org.eclipse.jgit.transport; +import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_1; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2; import java.io.EOFException; import java.io.IOException; @@ -21,20 +26,28 @@ import java.io.OutputStream; import java.text.MessageFormat; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.RemoteRepositoryException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.TemporaryBuffer; import org.eclipse.jgit.util.io.InterruptTimer; import org.eclipse.jgit.util.io.TimeoutInputStream; import org.eclipse.jgit.util.io.TimeoutOutputStream; @@ -86,17 +99,27 @@ abstract class BasePackConnection extends BaseConnection { protected boolean statelessRPC; /** Capability tokens advertised by the remote side. */ - private final Set remoteCapablities = new HashSet<>(); + private final Map remoteCapabilities = new HashMap<>(); /** Extra objects the remote has, but which aren't offered as refs. */ protected final Set additionalHaves = new HashSet<>(); + private TransferConfig.ProtocolVersion protocol = TransferConfig.ProtocolVersion.V0; + BasePackConnection(PackTransport packTransport) { transport = (Transport) packTransport; local = transport.local; uri = transport.uri; } + TransferConfig.ProtocolVersion getProtocolVersion() { + return protocol; + } + + void setProtocolVersion(@NonNull TransferConfig.ProtocolVersion protocol) { + this.protocol = protocol; + } + /** * Configure this connection with the directional pipes. * @@ -141,12 +164,15 @@ protected final void init(InputStream myIn, OutputStream myOut) { * {@link #close()} and the exception is wrapped (if necessary) and thrown * as a {@link org.eclipse.jgit.errors.TransportException}. * + * @return {@code true} if the refs were read; {@code false} otherwise + * indicating that {@link #lsRefs} must be called + * * @throws org.eclipse.jgit.errors.TransportException * the reference list could not be scanned. */ - protected void readAdvertisedRefs() throws TransportException { + protected boolean readAdvertisedRefs() throws TransportException { try { - readAdvertisedRefsImpl(); + return readAdvertisedRefsImpl(); } catch (TransportException err) { close(); throw err; @@ -156,35 +182,66 @@ protected void readAdvertisedRefs() throws TransportException { } } - private void readAdvertisedRefsImpl() throws IOException { + private String readLine() throws IOException { + String line = pckIn.readString(); + if (PacketLineIn.isEnd(line)) { + return null; + } + if (line.startsWith("ERR ")) { //$NON-NLS-1$ + // This is a customized remote service error. + // Users should be informed about it. + throw new RemoteRepositoryException(uri, line.substring(4)); + } + return line; + } + + private boolean readAdvertisedRefsImpl() throws IOException { final LinkedHashMap avail = new LinkedHashMap<>(); - for (;;) { + for (boolean first = true;; first = false) { String line; - try { - line = pckIn.readString(); - } catch (EOFException eof) { - if (avail.isEmpty()) + if (first) { + boolean isV1 = false; + try { + line = readLine(); + } catch (EOFException e) { throw noRepository(); - throw eof; - } - if (PacketLineIn.isEnd(line)) - break; - - if (line.startsWith("ERR ")) { //$NON-NLS-1$ - // This is a customized remote service error. - // Users should be informed about it. - throw new RemoteRepositoryException(uri, line.substring(4)); - } - - if (avail.isEmpty()) { + } + if (line != null && VERSION_1.equals(line)) { + // Same as V0, except for this extra line. We shouldn't get + // it since we never request V1. + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + isV1 = true; + line = readLine(); + } + if (line == null) { + break; + } final int nul = line.indexOf('\0'); if (nul >= 0) { - // The first line (if any) may contain "hidden" - // capability values after a NUL byte. - remoteCapablities.addAll( - Arrays.asList(line.substring(nul + 1).split(" "))); //$NON-NLS-1$ + // Protocol V0: The first line (if any) may contain + // "hidden" capability values after a NUL byte. + for (String capability : line.substring(nul + 1) + .split(" ")) { //$NON-NLS-1$ + addCapability(capability); + } line = line.substring(0, nul); + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + } else if (!isV1 && VERSION_2.equals(line)) { + // Protocol V2: remaining lines are capabilities as + // key=value pairs + setProtocolVersion(TransferConfig.ProtocolVersion.V2); + readCapabilitiesV2(); + // Break out here so that stateless RPC transports get a + // chance to set up the output stream. + return false; + } else { + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + } + } else { + line = readLine(); + if (line == null) { + break; } } @@ -193,42 +250,228 @@ private void readAdvertisedRefsImpl() throws IOException { throw invalidRefAdvertisementLine(line); } String name = line.substring(41, line.length()); - if (avail.isEmpty() && name.equals("capabilities^{}")) { //$NON-NLS-1$ - // special line from git-receive-pack to show + if (first && name.equals("capabilities^{}")) { //$NON-NLS-1$ + // special line from git-receive-pack (protocol V0) to show // capabilities when there are no refs to advertise + // TODO: throw error if protocol version is >= V2? continue; } - final ObjectId id; - try { - id = ObjectId.fromString(line.substring(0, 40)); - } catch (InvalidObjectIdException e) { - PackProtocolException ppe = invalidRefAdvertisementLine(line); - ppe.initCause(e); - throw ppe; - } + final ObjectId id = toId(line, line.substring(0, 40)); if (name.equals(".have")) { //$NON-NLS-1$ additionalHaves.add(id); - } else if (name.endsWith("^{}")) { //$NON-NLS-1$ - name = name.substring(0, name.length() - 3); - final Ref prior = avail.get(name); - if (prior == null) - throw new PackProtocolException(uri, MessageFormat.format( - JGitText.get().advertisementCameBefore, name, name)); - - if (prior.getPeeledObjectId() != null) - throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$ - - avail.put(name, new ObjectIdRef.PeeledTag( - Ref.Storage.NETWORK, name, prior.getObjectId(), id)); } else { - final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag( - Ref.Storage.NETWORK, name, id)); - if (prior != null) - throw duplicateAdvertisement(name); + processLineV1(name, id, avail); } } available(avail); + return true; + } + + /** + * Issue a protocol V2 ls-refs command and read its response. + * + * @param refSpecs + * to produce ref prefixes from if the server supports git + * protocol V2 + * @param additionalPatterns + * to use for ref prefixes if the server supports git protocol V2 + * @throws TransportException + * if the command could not be run or its output not be read + */ + protected void lsRefs(Collection refSpecs, + String... additionalPatterns) throws TransportException { + try { + lsRefsImpl(refSpecs, additionalPatterns); + } catch (TransportException err) { + close(); + throw err; + } catch (IOException | RuntimeException err) { + close(); + throw new TransportException(err.getMessage(), err); + } + } + + private void lsRefsImpl(Collection refSpecs, + String... additionalPatterns) throws IOException { + TemporaryBuffer state = null; + PacketLineOut pckState = null; + PacketLineOut output = pckOut; + if (statelessRPC) { + state = new TemporaryBuffer.Heap(Integer.MAX_VALUE); + pckState = new PacketLineOut(state); + output = pckState; + } + output.writeString("command=" + COMMAND_LS_REFS); //$NON-NLS-1$ + // Add the user-agent + String agent = UserAgent.get(); + if (agent != null && isCapableOf(OPTION_AGENT)) { + output.writeString(OPTION_AGENT + '=' + agent); + } + output.writeDelim(); + output.writeString("peel"); //$NON-NLS-1$ + for (String refPrefix : getRefPrefixes(refSpecs, additionalPatterns)) { + output.writeString("ref-prefix " + refPrefix); //$NON-NLS-1$ + } + output.end(); + output.flush(); + if (state != null) { + state.writeTo(out, null); + out.flush(); + state = null; + pckState = null; + } + final LinkedHashMap avail = new LinkedHashMap<>(); + for (;;) { + String line = readLine(); + if (line == null) { + break; + } + // Expecting to get a line in the form "sha1 refname" + if (line.length() < 41 || line.charAt(40) != ' ') { + throw invalidRefAdvertisementLine(line); + } + String name = line.substring(41, line.length()); + final ObjectId id = toId(line, line.substring(0, 40)); + if (name.equals(".have")) { //$NON-NLS-1$ + additionalHaves.add(id); + } else { + processLineV2(line, id, name, avail); + } + } + available(avail); + } + + private Collection getRefPrefixes(Collection refSpecs, + String... additionalPatterns) { + if (refSpecs.isEmpty() && (additionalPatterns == null + || additionalPatterns.length == 0)) { + return Collections.emptyList(); + } + Set patterns = new HashSet<>(); + if (additionalPatterns != null) { + Arrays.stream(additionalPatterns).filter(Objects::nonNull) + .forEach(patterns::add); + } + for (RefSpec spec : refSpecs) { + // TODO: for now we only do protocol V2 for fetch. For push + // RefSpecs, the logic would need to be different. (At the + // minimum, take spec.getDestination().) + String src = spec.getSource(); + if (ObjectId.isId(src)) { + continue; + } + if (spec.isWildcard()) { + patterns.add(src.substring(0, src.indexOf('*'))); + } else { + patterns.add(src); + patterns.add(Constants.R_REFS + src); + patterns.add(Constants.R_HEADS + src); + patterns.add(Constants.R_TAGS + src); + } + } + return patterns; + } + + private void readCapabilitiesV2() throws IOException { + // In git protocol V2, capabilities are different. If it's a key-value + // pair, the key may be a command name, and the value a space-separated + // list of capabilities for that command. We still store it in the same + // map as for protocol v0/v1. Protocol v2 code has to account for this. + for (;;) { + String line = readLine(); + if (line == null) { + break; + } + addCapability(line); + } + } + + private void addCapability(String capability) { + String parts[] = capability.split("=", 2); //$NON-NLS-1$ + if (parts.length == 2) { + remoteCapabilities.put(parts[0], parts[1]); + } + remoteCapabilities.put(capability, null); + } + + private ObjectId toId(String line, String value) + throws PackProtocolException { + try { + return ObjectId.fromString(value); + } catch (InvalidObjectIdException e) { + PackProtocolException ppe = invalidRefAdvertisementLine(line); + ppe.initCause(e); + throw ppe; + } + } + + private void processLineV1(String name, ObjectId id, Map avail) + throws IOException { + if (name.endsWith("^{}")) { //$NON-NLS-1$ + name = name.substring(0, name.length() - 3); + final Ref prior = avail.get(name); + if (prior == null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().advertisementCameBefore, name, name)); + } + if (prior.getPeeledObjectId() != null) { + throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$ + } + avail.put(name, new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, + prior.getObjectId(), id)); + } else { + final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag( + Ref.Storage.NETWORK, name, id)); + if (prior != null) { + throw duplicateAdvertisement(name); + } + } + } + + private void processLineV2(String line, ObjectId id, String rest, + Map avail) throws IOException { + String[] parts = rest.split(" "); //$NON-NLS-1$ + String name = parts[0]; + // Two attributes possible, symref-target or peeled + String symRefTarget = null; + String peeled = null; + for (int i = 1; i < parts.length; i++) { + if (parts[i].startsWith(REF_ATTR_SYMREF_TARGET)) { + if (symRefTarget != null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().duplicateRefAttribute, line)); + } + symRefTarget = parts[i] + .substring(REF_ATTR_SYMREF_TARGET.length()); + } else if (parts[i].startsWith(REF_ATTR_PEELED)) { + if (peeled != null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().duplicateRefAttribute, line)); + } + peeled = parts[i].substring(REF_ATTR_PEELED.length()); + } + if (peeled != null && symRefTarget != null) { + break; + } + } + Ref idRef; + if (peeled != null) { + idRef = new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, id, + toId(line, peeled)); + } else { + idRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, name, id); + } + Ref prior = avail.put(name, idRef); + if (prior != null) { + throw duplicateAdvertisement(name); + } + if (symRefTarget != null) { + // What do we have to do with this information? In protocol V0, this + // info comes in the capability advertisement, but OPTION_SYMREF is + // used only in RefAdvertiser. JGit's client side doesn't make use + // of this information, and actually never requests it. + } } /** @@ -252,7 +495,7 @@ protected TransportException noRepository() { * @return whether this option is supported */ protected boolean isCapableOf(String option) { - return remoteCapablities.contains(option); + return remoteCapabilities.containsKey(option); } /** @@ -272,6 +515,17 @@ protected boolean wantCapability(StringBuilder b, String option) { return true; } + /** + * Return a capability value. + * + * @param option + * to get + * @return the value stored, if any. + */ + protected String getCapability(String option) { + return remoteCapabilities.get(option); + } + /** * Add user agent capability * @@ -280,7 +534,7 @@ protected boolean wantCapability(StringBuilder b, String option) { */ protected void addUserAgentCapability(StringBuilder b) { String a = UserAgent.get(); - if (a != null && UserAgent.hasAgent(remoteCapablities)) { + if (a != null && remoteCapabilities.get(OPTION_AGENT) != null) { b.append(' ').append(OPTION_AGENT).append('=').append(a); } } @@ -288,7 +542,8 @@ protected void addUserAgentCapability(StringBuilder b) { /** {@inheritDoc} */ @Override public String getPeerUserAgent() { - return UserAgent.getAgent(remoteCapablities, super.getPeerUserAgent()); + String agent = remoteCapabilities.get(OPTION_AGENT); + return agent != null ? agent : super.getPeerUserAgent(); } private PackProtocolException duplicateAdvertisement(String name) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index a2fb51f46..2432641ed 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -16,9 +16,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; import org.eclipse.jgit.errors.PackProtocolException; @@ -212,6 +215,8 @@ public abstract class BasePackFetchConnection extends BasePackConnection private PacketLineOut pckState; + private Set fetchCapabilities = new HashSet<>(); + /** * Either FilterSpec.NO_FILTER for a filter that doesn't filter * anything, or a filter that indicates what and what not to send to the @@ -354,6 +359,28 @@ protected void doFetch(final ProgressMonitor monitor, pckState = new PacketLineOut(state); } + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + sideband = true; + noDone = true; + multiAck = MultiAck.DETAILED; + setFetchOptions(); + PacketLineOut output = statelessRPC ? pckState : pckOut; + output.writeString( + "command=" + GitProtocolConstants.COMMAND_FETCH); //$NON-NLS-1$ + // Capabilities are sent as command arguments in protocol V2 + String agent = UserAgent.get(); + if (agent != null + && isCapableOf(GitProtocolConstants.OPTION_AGENT)) { + output.writeString( + GitProtocolConstants.OPTION_AGENT + '=' + agent); + } + output.writeDelim(); + // Arguments + for (String capability : getCapabilitiesV2()) { + output.writeString(capability); + } + } if (sendWants(want)) { negotiate(monitor); @@ -362,6 +389,25 @@ protected void doFetch(final ProgressMonitor monitor, state = null; pckState = null; + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + String header = pckIn.readString(); + if (PacketLineIn.isEnd(header)) { + // No packfile following. + return; + } + + if (header.startsWith("ERR ")) { //$NON-NLS-1$ + // Protocol V2 may give us an error here (for instance, + // invalid want) + throw new PackProtocolException(header.substring(4)); + } + if (!GitProtocolConstants.SECTION_PACKFILE.equals(header)) { + throw new PackProtocolException(MessageFormat.format( + JGitText.get().expectedGot, + GitProtocolConstants.SECTION_PACKFILE, header)); + } + } receivePack(monitor, outputStream); } } catch (CancelledException ce) { @@ -381,6 +427,14 @@ public void close() { super.close(); } + private void setFetchOptions() { + String advertised = getCapability(GitProtocolConstants.COMMAND_FETCH); + if (advertised == null) { + return; + } + fetchCapabilities.addAll(Arrays.asList(advertised.split(" "))); //$NON-NLS-1$ + } + FetchConfig getFetchConfig() { return local.getConfig().get(FetchConfig::new); } @@ -479,10 +533,11 @@ private boolean sendWants(Collection want) throws IOException { final StringBuilder line = new StringBuilder(46); line.append("want "); //$NON-NLS-1$ line.append(objectId.name()); - if (first) { + if (first && TransferConfig.ProtocolVersion.V0 + .equals(getProtocolVersion())) { line.append(enableCapabilities()); - first = false; } + first = false; line.append('\n'); p.writeString(line.toString()); } @@ -492,11 +547,35 @@ private boolean sendWants(Collection want) throws IOException { if (!filterSpec.isNoOp()) { p.writeString(filterSpec.filterLine()); } - p.end(); - outNeedsEnd = false; + if (TransferConfig.ProtocolVersion.V0.equals(getProtocolVersion())) { + p.end(); + outNeedsEnd = false; + } return true; } + private Set getCapabilitiesV2() throws TransportException { + Set capabilities = new LinkedHashSet<>(); + if (noProgress) { + capabilities.add(OPTION_NO_PROGRESS); + } + if (includeTags) { + capabilities.add(OPTION_INCLUDE_TAG); + } + if (allowOfsDelta) { + capabilities.add(OPTION_OFS_DELTA); + } + if (thinPack) { + capabilities.add(OPTION_THIN_PACK); + } + if (!filterSpec.isNoOp() + && !fetchCapabilities.contains(OPTION_FILTER)) { + throw new PackProtocolException(uri, + JGitText.get().filterRequiresCapability); + } + return capabilities; + } + private String enableCapabilities() throws TransportException { final StringBuilder line = new StringBuilder(); if (noProgress) @@ -550,6 +629,7 @@ private void negotiate(ProgressMonitor monitor) throws IOException, boolean receivedContinue = false; boolean receivedAck = false; boolean receivedReady = false; + boolean needsAcknowledgementV2 = true; if (statelessRPC) { state.writeTo(out, null); @@ -563,7 +643,7 @@ private void negotiate(ProgressMonitor monitor) throws IOException, } ObjectId o = c.getId(); - pckOut.writeString("have " + o.name() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ + pckOut.writeString("have " + o.name() + '\n'); //$NON-NLS-1$ havesSent++; havesSinceLastContinue++; @@ -580,6 +660,7 @@ private void negotiate(ProgressMonitor monitor) throws IOException, } pckOut.end(); + outNeedsEnd = false; resultsPending++; // Each end will cause a result to come back. if (havesSent == 32 && !statelessRPC) { @@ -591,9 +672,32 @@ private void negotiate(ProgressMonitor monitor) throws IOException, continue; } + // Read the section header + if (needsAcknowledgementV2 && TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + String header = pckIn.readString(); + if (!GitProtocolConstants.SECTION_ACKNOWLEDGMENTS + .equals(header)) { + throw new PackProtocolException(MessageFormat.format( + JGitText.get().expectedGot, + GitProtocolConstants.SECTION_ACKNOWLEDGMENTS, + header)); + } + needsAcknowledgementV2 = false; + } READ_RESULT: for (;;) { - final AckNackResult anr = pckIn.readACK(ackId); + AckNackResult anr = pckIn.readACKorEOF(ackId); switch (anr) { + case ACK_EOF: + if (TransferConfig.ProtocolVersion.V0 + .equals(getProtocolVersion())) { + throw new PackProtocolException( + JGitText.get().expectedACKNAKFoundEOF); + } + // More lines needed + resultsPending--; + break READ_RESULT; + case NAK: // More have lines are necessary to compute the // pack on the remote side. Keep doing that. @@ -602,17 +706,24 @@ private void negotiate(ProgressMonitor monitor) throws IOException, break READ_RESULT; case ACK: - // The remote side is happy and knows exactly what - // to send us. There is no further negotiation and - // we can break out immediately. - // - multiAck = MultiAck.OFF; - resultsPending = 0; - receivedAck = true; - if (statelessRPC) { - state.writeTo(out, null); + if (TransferConfig.ProtocolVersion.V0 + .equals(getProtocolVersion())) { + // The remote side is happy and knows exactly what + // to send us. There is no further negotiation and + // we can break out immediately. + // + multiAck = MultiAck.OFF; + resultsPending = 0; + receivedAck = true; + if (statelessRPC) { + state.writeTo(out, null); + } + break SEND_HAVES; } - break SEND_HAVES; + // Keep on reading ACKs until we get the ACK_READY + markCommon(walk.parseAny(ackId), AckNackResult.ACK_COMMON); + receivedAck = true; + break; case ACK_CONTINUE: case ACK_COMMON: @@ -628,6 +739,12 @@ private void negotiate(ProgressMonitor monitor) throws IOException, havesSinceLastContinue = 0; if (anr == AckNackResult.ACK_READY) { receivedReady = true; + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + multiAck = MultiAck.OFF; + resultsPending = 0; + break SEND_HAVES; + } } break; } @@ -661,15 +778,36 @@ private void negotiate(ProgressMonitor monitor) throws IOException, throw new CancelledException(); } + if (receivedReady && TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + // We'll get the packfile right away. Skip the delimiter. + String delim = pckIn.readString(); + if (!PacketLineIn.isDelimiter(delim)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0001", delim)); //$NON-NLS-1$ + } + return; + } if (!receivedReady || !noDone) { // When statelessRPC is true we should always leave SEND_HAVES // loop above while in the middle of a request. This allows us // to just write done immediately. // pckOut.writeString("done\n"); //$NON-NLS-1$ + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + pckOut.end(); + outNeedsEnd = false; + pckOut.flush(); + // Protocol V2 will skip acknowledgments completely if we send + // a done. + return; + } pckOut.flush(); } + // Below is protocol V0 only. + if (!receivedAck) { // Apparently if we have never received an ACK earlier // there is one more result expected from the done we @@ -689,6 +827,14 @@ private void negotiate(ProgressMonitor monitor) throws IOException, // break; + case ACK_EOF: + if (TransferConfig.ProtocolVersion.V0 + .equals(getProtocolVersion())) { + throw new PackProtocolException( + JGitText.get().expectedACKNAKFoundEOF); + } + break READ_RESULT; + case ACK: // A solitary ACK at this point means the remote won't // speak anymore, but is going to send us a pack now. @@ -697,7 +843,16 @@ private void negotiate(ProgressMonitor monitor) throws IOException, case ACK_CONTINUE: case ACK_COMMON: + // We will expect a normal ACK to break out of the loop. + // + multiAck = MultiAck.CONTINUE; + break; + case ACK_READY: + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + break READ_RESULT; + } // We will expect a normal ACK to break out of the loop. // multiAck = MultiAck.CONTINUE; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java index 0f1892a97..b9b764c9d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java @@ -102,7 +102,21 @@ void execute(ProgressMonitor monitor, FetchResult result) private void executeImp(final ProgressMonitor monitor, final FetchResult result) throws NotSupportedException, TransportException { - conn = transport.openFetch(); + final TagOpt tagopt = transport.getTagOpt(); + String getTags = (tagopt == TagOpt.NO_TAGS) ? null : Constants.R_TAGS; + String getHead = null; + try { + // If we don't have a HEAD yet, we're cloning and need to get the + // upstream HEAD, too. + Ref head = transport.local.exactRef(Constants.HEAD); + ObjectId id = head != null ? head.getObjectId() : null; + if (id == null || id.equals(ObjectId.zeroId())) { + getHead = Constants.HEAD; + } + } catch (IOException e) { + // Ignore + } + conn = transport.openFetch(toFetch, getTags, getHead); try { result.setAdvertisedRefs(transport.getURI(), conn.getRefsMap()); result.peerUserAgent = conn.getPeerUserAgent(); @@ -119,7 +133,6 @@ private void executeImp(final ProgressMonitor monitor, } Collection additionalTags = Collections. emptyList(); - final TagOpt tagopt = transport.getTagOpt(); if (tagopt == TagOpt.AUTO_FOLLOW) additionalTags = expandAutoFollowTags(); else if (tagopt == TagOpt.FETCH_TAGS) @@ -253,7 +266,17 @@ private void reopenConnection() throws NotSupportedException, if (conn != null) return; - conn = transport.openFetch(); + // Build prefixes + Set prefixes = new HashSet<>(); + for (Ref toGet : askFor.values()) { + String src = toGet.getName(); + prefixes.add(src); + prefixes.add(Constants.R_REFS + src); + prefixes.add(Constants.R_HEADS + src); + prefixes.add(Constants.R_TAGS + src); + } + conn = transport.openFetch(Collections.emptyList(), + prefixes.toArray(new String[0])); // Since we opened a new connection we cannot be certain // that the system we connected to has the same exact set diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java index 35e2978bc..ea6dec8ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java @@ -247,6 +247,74 @@ public final class GitProtocolConstants { */ public static final String COMMAND_FETCH = "fetch"; //$NON-NLS-1$ + /** + * HTTP header to set by clients to request a specific git protocol version + * in the HTTP transport. + * + * @since 5.10 + */ + public static final String PROTOCOL_HEADER = "Git-Protocol"; //$NON-NLS-1$ + + /** + * Environment variable to set by clients to request a specific git protocol + * in the file:// and ssh:// transports. + * + * @since 5.10 + */ + public static final String PROTOCOL_ENVIRONMENT_VARIABLE = "GIT_PROTOCOL"; //$NON-NLS-1$ + + /** + * Protocol V2 ref advertisement attribute containing the peeled object id + * for annotated tags. + * + * @since 5.10 + */ + public static final String REF_ATTR_PEELED = "peeled:"; //$NON-NLS-1$ + + /** + * Protocol V2 ref advertisement attribute containing the name of the ref + * for symbolic refs. + * + * @since 5.10 + */ + public static final String REF_ATTR_SYMREF_TARGET = "symref-target:"; //$NON-NLS-1$ + + /** + * Protocol V2 acknowledgments section header. + * + * @since 5.10 + */ + public static final String SECTION_ACKNOWLEDGMENTS = "acknowledgments"; //$NON-NLS-1$ + + /** + * Protocol V2 packfile section header. + * + * @since 5.10 + */ + public static final String SECTION_PACKFILE = "packfile"; //$NON-NLS-1$ + + /** + * Protocol announcement for protocol version 1. This is the same as V0, + * except for this initial line. + * + * @since 5.10 + */ + public static final String VERSION_1 = "version 1"; //$NON-NLS-1$ + + /** + * Protocol announcement for protocol version 2. + * + * @since 5.10 + */ + public static final String VERSION_2 = "version 2"; //$NON-NLS-1$ + + /** + * Protocol request for protocol version 2. + * + * @since 5.10 + */ + public static final String VERSION_2_REQUEST = "version=2"; //$NON-NLS-1$ + enum MultiAck { OFF, CONTINUE, DETAILED; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index 52a5576e4..32046dc43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -72,7 +72,9 @@ enum AckNackResult { /** ACK + common */ ACK_COMMON, /** ACK + ready */ - ACK_READY; + ACK_READY, + /** EOF (0000) recieved in protocol V2 when expecting an ACK/NAK */ + ACK_EOF; } private final byte[] lineBuffer = new byte[SideBandOutputStream.SMALL_BUF]; @@ -103,12 +105,26 @@ public PacketLineIn(InputStream in, long limit) { this.limit = limit; } + AckNackResult readACK(MutableObjectId returnedId) throws IOException { + AckNackResult result = readACKorEOF(returnedId); + if (result == AckNackResult.ACK_EOF) { + throw new PackProtocolException( + JGitText.get().expectedACKNAKFoundEOF); + } + return result; + } + + AckNackResult readACKorEOF(MutableObjectId returnedId) throws IOException { final String line = readString(); if (line.length() == 0) - throw new PackProtocolException(JGitText.get().expectedACKNAKFoundEOF); + return AckNackResult.ACK_EOF; if ("NAK".equals(line)) //$NON-NLS-1$ return AckNackResult.NAK; + if ("ready".equals(line)) { //$NON-NLS-1$ + // Protocol V2 + return AckNackResult.ACK_READY; + } if (line.startsWith("ACK ")) { //$NON-NLS-1$ returnedId.fromString(line.substring(4, 44)); if (line.length() == 44) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java index 3adebba03..3f50c5784 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java @@ -13,6 +13,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SYMREF; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET; import java.io.IOException; import java.nio.ByteBuffer; @@ -287,7 +289,8 @@ public Set send(Collection refs) throws IOException { if (useProtocolV2) { String symrefPart = symrefs.containsKey(ref.getName()) - ? (" symref-target:" + symrefs.get(ref.getName())) //$NON-NLS-1$ + ? (' ' + REF_ATTR_SYMREF_TARGET + + symrefs.get(ref.getName())) : ""; //$NON-NLS-1$ String peelPart = ""; //$NON-NLS-1$ if (derefTags) { @@ -296,7 +299,8 @@ public Set send(Collection refs) throws IOException { } ObjectId peeledObjectId = ref.getPeeledObjectId(); if (peeledObjectId != null) { - peelPart = " peeled:" + peeledObjectId.getName(); //$NON-NLS-1$ + peelPart = ' ' + REF_ATTR_PEELED + + peeledObjectId.getName(); } } writeOne(objectId.getName() + " " + ref.getName() + symrefPart //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java new file mode 100644 index 000000000..8119f75aa --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2020, Thomas Wolf + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.transport; + +import java.io.IOException; +import java.util.Map; + +/** + * A {@link RemoteSession} that supports passing environment variables to + * commands. + * + * @since 5.10 + */ +public interface RemoteSession2 extends RemoteSession { + + /** + * Creates a new remote {@link Process} to execute the given command. The + * returned process's streams exist and are connected, and execution of the + * process is already started. + * + * @param commandName + * command to execute + * @param environment + * environment variables to pass on + * @param timeout + * timeout value, in seconds, for creating the remote process + * @return a new remote process, already started + * @throws java.io.IOException + * may be thrown in several cases. For example, on problems + * opening input or output streams or on problems connecting or + * communicating with the remote host. For the latter two cases, + * a TransportException may be thrown (a subclass of + * java.io.IOException). + */ + Process exec(String commandName, Map environment, + int timeout) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index 2ddd0a612..f0f9e1db6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -39,6 +39,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.AbortedByHookException; import org.eclipse.jgit.errors.NotSupportedException; import org.eclipse.jgit.errors.TransportException; @@ -774,6 +775,10 @@ private static String findTrackingRefName(final String remoteName, private PrintStream hookOutRedirect; private PrePushHook prePush; + + @Nullable + TransferConfig.ProtocolVersion protocol; + /** * Create a new transport instance. * @@ -789,6 +794,7 @@ protected Transport(Repository local, URIish uri) { final TransferConfig tc = local.getConfig().get(TransferConfig.KEY); this.local = local; this.uri = uri; + this.protocol = tc.protocolVersion; this.objectChecker = tc.newObjectChecker(); this.credentialsProvider = CredentialsProvider.getDefault(); prePush = Hooks.prePush(local, hookOutRedirect); @@ -1452,6 +1458,43 @@ public Collection findRemoteRefUpdatesFor( public abstract FetchConnection openFetch() throws NotSupportedException, TransportException; + /** + * Begins a new connection for fetching from the remote repository. + *

+ * If the transport has no local repository, the fetch connection can only + * be used for reading remote refs. + *

+ *

+ * If the server supports git protocol V2, the {@link RefSpec}s and the + * additional patterns, if any, are used to restrict the server's ref + * advertisement to matching refs only. + *

+ *

+ * Transports that want to support git protocol V2 must override + * this; the default implementation ignores its arguments and calls + * {@link #openFetch()}. + *

+ * + * @param refSpecs + * that will be fetched via + * {@link FetchConnection#fetch(ProgressMonitor, Collection, java.util.Set, OutputStream)} later + * @param additionalPatterns + * that will be set as ref prefixes if the server supports git + * protocol V2; {@code null} values are ignored + * + * @return a fresh connection to fetch from the remote repository. + * @throws org.eclipse.jgit.errors.NotSupportedException + * the implementation does not support fetching. + * @throws org.eclipse.jgit.errors.TransportException + * the remote connection could not be established. + * @since 5.10 + */ + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return openFetch(); + } + /** * Begins a new connection for pushing into the remote repository. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java index 820ec1a67..5913aa4e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.UnknownHostException; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.Set; @@ -94,6 +95,13 @@ public FetchConnection openFetch() throws TransportException { return new TcpFetchConnection(); } + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return new TcpFetchConnection(refSpecs, additionalPatterns); + } + /** {@inheritDoc} */ @Override public PushConnection openPush() throws TransportException { @@ -130,7 +138,8 @@ Socket openConnection() throws TransportException { return s; } - void service(String name, PacketLineOut pckOut) + void service(String name, PacketLineOut pckOut, + TransferConfig.ProtocolVersion gitProtocol) throws IOException { final StringBuilder cmd = new StringBuilder(); cmd.append(name); @@ -144,6 +153,11 @@ void service(String name, PacketLineOut pckOut) cmd.append(uri.getPort()); } cmd.append('\0'); + if (TransferConfig.ProtocolVersion.V2.equals(gitProtocol)) { + cmd.append('\0'); + cmd.append(GitProtocolConstants.VERSION_2_REQUEST); + cmd.append('\0'); + } pckOut.writeString(cmd.toString()); pckOut.flush(); } @@ -152,6 +166,11 @@ class TcpFetchConnection extends BasePackFetchConnection { private Socket sock; TcpFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + TcpFetchConnection(Collection refSpecs, + String... additionalPatterns) throws TransportException { super(TransportGitAnon.this); sock = openConnection(); try { @@ -162,13 +181,19 @@ class TcpFetchConnection extends BasePackFetchConnection { sOut = new BufferedOutputStream(sOut); init(sIn, sOut); - service("git-upload-pack", pckOut); //$NON-NLS-1$ + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + service("git-upload-pack", pckOut, gitProtocol); //$NON-NLS-1$ } catch (IOException err) { close(); throw new TransportException(uri, JGitText.get().remoteHungUpUnexpectedly, err); } - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } @Override @@ -201,7 +226,7 @@ class TcpPushConnection extends BasePackPushConnection { sOut = new BufferedOutputStream(sOut); init(sIn, sOut); - service("git-receive-pack", pckOut); //$NON-NLS-1$ + service("git-receive-pack", pckOut, null); //$NON-NLS-1$ } catch (IOException err) { close(); throw new TransportException(uri, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java index b9cb2484d..6aa2e0bed 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java @@ -19,11 +19,13 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import org.eclipse.jgit.errors.NoRemoteRepositoryException; @@ -144,6 +146,13 @@ public FetchConnection openFetch() throws TransportException { return new SshFetchConnection(); } + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return new SshFetchConnection(refSpecs, additionalPatterns); + } + /** {@inheritDoc} */ @Override public PushConnection openPush() throws TransportException { @@ -196,29 +205,38 @@ private static boolean useExtSession() { return SystemReader.getInstance().getenv("GIT_SSH") != null; //$NON-NLS-1$ } - private class ExtSession implements RemoteSession { + private class ExtSession implements RemoteSession2 { + @Override public Process exec(String command, int timeout) throws TransportException { + return exec(command, null, timeout); + } + + @Override + public Process exec(String command, Map environment, + int timeout) throws TransportException { String ssh = SystemReader.getInstance().getenv("GIT_SSH"); //$NON-NLS-1$ boolean putty = ssh.toLowerCase(Locale.ROOT).contains("plink"); //$NON-NLS-1$ List args = new ArrayList<>(); args.add(ssh); - if (putty - && !ssh.toLowerCase(Locale.ROOT).contains("tortoiseplink")) //$NON-NLS-1$ + if (putty && !ssh.toLowerCase(Locale.ROOT) + .contains("tortoiseplink")) {//$NON-NLS-1$ args.add("-batch"); //$NON-NLS-1$ + } if (0 < getURI().getPort()) { args.add(putty ? "-P" : "-p"); //$NON-NLS-1$ //$NON-NLS-2$ args.add(String.valueOf(getURI().getPort())); } - if (getURI().getUser() != null) + if (getURI().getUser() != null) { args.add(getURI().getUser() + "@" + getURI().getHost()); //$NON-NLS-1$ - else + } else { args.add(getURI().getHost()); + } args.add(command); - ProcessBuilder pb = createProcess(args); + ProcessBuilder pb = createProcess(args, environment); try { return pb.start(); } catch (IOException err) { @@ -226,9 +244,13 @@ public Process exec(String command, int timeout) } } - private ProcessBuilder createProcess(List args) { + private ProcessBuilder createProcess(List args, + Map environment) { ProcessBuilder pb = new ProcessBuilder(); pb.command(args); + if (environment != null) { + pb.environment().putAll(environment); + } File directory = local != null ? local.getDirectory() : null; if (directory != null) { pb.environment().put(Constants.GIT_DIR_KEY, @@ -249,10 +271,31 @@ class SshFetchConnection extends BasePackFetchConnection { private StreamCopyThread errorThread; SshFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + SshFetchConnection(Collection refSpecs, + String... additionalPatterns) throws TransportException { super(TransportGitSsh.this); try { - process = getSession().exec(commandFor(getOptionUploadPack()), - getTimeout()); + RemoteSession session = getSession(); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + if (session instanceof RemoteSession2 + && TransferConfig.ProtocolVersion.V2 + .equals(gitProtocol)) { + process = ((RemoteSession2) session).exec( + commandFor(getOptionUploadPack()), Collections + .singletonMap( + GitProtocolConstants.PROTOCOL_ENVIRONMENT_VARIABLE, + GitProtocolConstants.VERSION_2_REQUEST), + getTimeout()); + } else { + process = session.exec(commandFor(getOptionUploadPack()), + getTimeout()); + } final MessageWriter msg = new MessageWriter(); setMessageWriter(msg); @@ -272,7 +315,9 @@ class SshFetchConnection extends BasePackFetchConnection { } try { - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } catch (NoRemoteRepositoryException notFound) { final String msgs = getMessages(); checkExecFailure(process.exitValue(), getOptionUploadPack(), diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 6768387e6..77a7a4640 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -33,8 +33,8 @@ import static org.eclipse.jgit.util.HttpSupport.METHOD_GET; import static org.eclipse.jgit.util.HttpSupport.METHOD_POST; +import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -339,11 +339,15 @@ public void setUseSmartHttp(boolean on) { @SuppressWarnings("resource") // Closed by caller private FetchConnection getConnection(HttpConnection c, InputStream in, - String service) throws IOException { + String service, Collection refSpecs, + String... additionalPatterns) throws IOException { BaseConnection f; if (isSmartHttp(c, service)) { - readSmartHeaders(in, service); - f = new SmartHttpFetchConnection(in); + InputStream withMark = in.markSupported() ? in + : new BufferedInputStream(in); + readSmartHeaders(withMark, service); + f = new SmartHttpFetchConnection(withMark, refSpecs, + additionalPatterns); } else { // Assume this server doesn't support smart HTTP fetch // and fall back on dumb object walking. @@ -357,11 +361,23 @@ private FetchConnection getConnection(HttpConnection c, InputStream in, @Override public FetchConnection openFetch() throws TransportException, NotSupportedException { + return openFetch(Collections.emptyList()); + } + + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { final String service = SVC_UPLOAD_PACK; try { - final HttpConnection c = connect(service); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + HttpConnection c = connect(service, gitProtocol); try (InputStream in = openInputStream(c)) { - return getConnection(c, in, service); + return getConnection(c, in, service, refSpecs, + additionalPatterns); } } catch (NotSupportedException | TransportException err) { throw err; @@ -456,8 +472,9 @@ public PushConnection openPush() throws NotSupportedException, private PushConnection smartPush(String service, HttpConnection c, InputStream in) throws IOException, TransportException { - readSmartHeaders(in, service); - SmartHttpPushConnection p = new SmartHttpPushConnection(in); + BufferedInputStream inBuf = new BufferedInputStream(in); + readSmartHeaders(inBuf, service); + SmartHttpPushConnection p = new SmartHttpPushConnection(inBuf); p.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER)); return p; } @@ -494,6 +511,12 @@ private NoRemoteRepositoryException createNotFoundException(URIish u, private HttpConnection connect(String service) throws TransportException, NotSupportedException { + return connect(service, null); + } + + private HttpConnection connect(String service, + TransferConfig.ProtocolVersion protocolVersion) + throws TransportException, NotSupportedException { URL u = getServiceURL(service); int authAttempts = 1; int redirects = 0; @@ -507,6 +530,11 @@ private HttpConnection connect(String service) } else { conn.setRequestProperty(HDR_ACCEPT, "*/*"); //$NON-NLS-1$ } + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + conn.setRequestProperty( + GitProtocolConstants.PROTOCOL_HEADER, + GitProtocolConstants.VERSION_2_REQUEST); + } final int status = HttpSupport.response(conn); processResponseCookies(conn); switch (status) { @@ -1148,20 +1176,37 @@ private boolean isGzipContent(HttpConnection c) { private void readSmartHeaders(InputStream in, String service) throws IOException { - // A smart reply will have a '#' after the first 4 bytes, but - // a dumb reply cannot contain a '#' until after byte 41. Do a + // A smart protocol V0 reply will have a '#' after the first 4 bytes, + // but a dumb reply cannot contain a '#' until after byte 41. Do a // quick check to make sure its a smart reply before we parse // as a pkt-line stream. // - final byte[] magic = new byte[5]; + // There appears to be a confusion about this in protocol V2. Github + // sends the # service line as a git (not http) header also when + // protocol V2 is used. Gitlab also does so. JGit's UploadPack doesn't, + // and thus Gerrit also does not. + final byte[] magic = new byte[14]; + if (!in.markSupported()) { + throw new TransportException(uri, + JGitText.get().inputStreamMustSupportMark); + } + in.mark(14); IO.readFully(in, magic, 0, magic.length); + // Did we get 000dversion 2 or similar? (Canonical is 000eversion 2\n, + // but JGit and thus Gerrit omits the \n.) + if (Arrays.equals(Arrays.copyOfRange(magic, 4, 11), + "version".getBytes()) && magic[12] >= '1' && magic[12] <= '9') { //$NON-NLS-1$ + // It's a smart server doing version 1 or greater, but not sending + // the # service line header. Don't consume the version line. + in.reset(); + return; + } if (magic[4] != '#') { throw new TransportException(uri, MessageFormat.format( JGitText.get().expectedPktLineWithService, RawParseUtils.decode(magic))); } - - final PacketLineIn pckIn = new PacketLineIn(new UnionInputStream( - new ByteArrayInputStream(magic), in)); + in.reset(); + final PacketLineIn pckIn = new PacketLineIn(in); final String exp = "# service=" + service; //$NON-NLS-1$ final String act = pckIn.readString(); if (!exp.equals(act)) { @@ -1327,12 +1372,24 @@ class SmartHttpFetchConnection extends BasePackFetchConnection { SmartHttpFetchConnection(InputStream advertisement) throws TransportException { + this(advertisement, Collections.emptyList()); + } + + SmartHttpFetchConnection(InputStream advertisement, + Collection refSpecs, String... additionalPatterns) + throws TransportException { super(TransportHttp.this); statelessRPC = true; init(advertisement, DisabledOutputStream.INSTANCE); outNeedsEnd = false; - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + // Must be protocol V2 + LongPollService service = new LongPollService(SVC_UPLOAD_PACK, + getProtocolVersion()); + init(service.getInputStream(), service.getOutputStream()); + lsRefs(refSpecs, additionalPatterns); + } } @Override @@ -1340,7 +1397,8 @@ protected void doFetch(final ProgressMonitor monitor, final Collection want, final Set have, final OutputStream outputStream) throws TransportException { try { - svc = new MultiRequestService(SVC_UPLOAD_PACK); + svc = new MultiRequestService(SVC_UPLOAD_PACK, + getProtocolVersion()); init(svc.getInputStream(), svc.getOutputStream()); super.doFetch(monitor, want, have, outputStream); } finally { @@ -1369,7 +1427,8 @@ class SmartHttpPushConnection extends BasePackPushConnection { protected void doPush(final ProgressMonitor monitor, final Map refUpdates, OutputStream outputStream) throws TransportException { - final Service svc = new MultiRequestService(SVC_RECEIVE_PACK); + final Service svc = new MultiRequestService(SVC_RECEIVE_PACK, + getProtocolVersion()); init(svc.getInputStream(), svc.getOutputStream()); super.doPush(monitor, refUpdates, outputStream); } @@ -1389,10 +1448,14 @@ abstract class Service { protected final HttpExecuteStream execute; + protected final TransferConfig.ProtocolVersion protocolVersion; + final UnionInputStream in; - Service(String serviceName) { + Service(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { this.serviceName = serviceName; + this.protocolVersion = protocolVersion; this.requestType = "application/x-" + serviceName + "-request"; //$NON-NLS-1$ //$NON-NLS-2$ this.responseType = "application/x-" + serviceName + "-result"; //$NON-NLS-1$ //$NON-NLS-2$ @@ -1408,6 +1471,10 @@ void openStream() throws IOException { conn.setDoOutput(true); conn.setRequestProperty(HDR_CONTENT_TYPE, requestType); conn.setRequestProperty(HDR_ACCEPT, responseType); + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + conn.setRequestProperty(GitProtocolConstants.PROTOCOL_HEADER, + GitProtocolConstants.VERSION_2_REQUEST); + } } void sendRequest() throws IOException { @@ -1663,8 +1730,9 @@ protected OutputStream overflow() throws IOException { class MultiRequestService extends Service { boolean finalRequest; - MultiRequestService(String serviceName) { - super(serviceName); + MultiRequestService(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { + super(serviceName, protocolVersion); } /** Keep opening send-receive pairs to the given URI. */ @@ -1701,11 +1769,10 @@ void execute() throws IOException { /** Service for maintaining a single long-poll connection. */ class LongPollService extends Service { - /** - * @param serviceName - */ - LongPollService(String serviceName) { - super(serviceName); + + LongPollService(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { + super(serviceName, protocolVersion); } /** Only open one send-receive request. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java index 403f98d86..89cb65e9a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -153,11 +154,17 @@ private Repository openRepo() throws TransportException { /** {@inheritDoc} */ @Override public FetchConnection openFetch() throws TransportException { + return openFetch(Collections.emptyList()); + } + + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) throws TransportException { final String up = getOptionUploadPack(); if (!"git-upload-pack".equals(up) //$NON-NLS-1$ - && !"git upload-pack".equals(up)) //$NON-NLS-1$ - return new ForkLocalFetchConnection(); - + && !"git upload-pack".equals(up)) {//$NON-NLS-1$ + return new ForkLocalFetchConnection(refSpecs, additionalPatterns); + } UploadPackFactory upf = (Void req, Repository db) -> createUploadPack(db); return new InternalFetchConnection<>(this, upf, null, openRepo()); @@ -193,6 +200,23 @@ public void close() { */ protected Process spawn(String cmd) throws TransportException { + return spawn(cmd, null); + } + + /** + * Spawn process + * + * @param cmd + * command + * @param protocolVersion + * to use + * @return a {@link java.lang.Process} object. + * @throws org.eclipse.jgit.errors.TransportException + * if any. + */ + private Process spawn(String cmd, + TransferConfig.ProtocolVersion protocolVersion) + throws TransportException { try { String[] args = { "." }; //$NON-NLS-1$ ProcessBuilder proc = local.getFS().runInShell(cmd, args); @@ -208,7 +232,10 @@ protected Process spawn(String cmd) env.remove("GIT_GRAFT_FILE"); //$NON-NLS-1$ env.remove("GIT_INDEX_FILE"); //$NON-NLS-1$ env.remove("GIT_NO_REPLACE_OBJECTS"); //$NON-NLS-1$ - + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + env.put(GitProtocolConstants.PROTOCOL_ENVIRONMENT_VARIABLE, + GitProtocolConstants.VERSION_2_REQUEST); + } return proc.start(); } catch (IOException err) { throw new TransportException(uri, err.getMessage(), err); @@ -221,12 +248,21 @@ class ForkLocalFetchConnection extends BasePackFetchConnection { private Thread errorReaderThread; ForkLocalFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + ForkLocalFetchConnection(Collection refSpecs, + String... additionalPatterns) throws TransportException { super(TransportLocal.this); final MessageWriter msg = new MessageWriter(); setMessageWriter(msg); - uploadPack = spawn(getOptionUploadPack()); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + uploadPack = spawn(getOptionUploadPack(), gitProtocol); final InputStream upErr = uploadPack.getErrorStream(); errorReaderThread = new StreamCopyThread(upErr, msg.getRawStream()); @@ -239,7 +275,9 @@ class ForkLocalFetchConnection extends BasePackFetchConnection { upOut = new BufferedOutputStream(upOut); init(upIn, upOut); - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 1242ef1b4..9eefea0c4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -33,6 +33,7 @@ import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND_64K; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_THIN_PACK; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2_REQUEST; import static org.eclipse.jgit.util.RefMap.toRefMap; import java.io.ByteArrayOutputStream; @@ -709,7 +710,7 @@ public boolean isSideBand() throws RequestNotYetReadException { * @since 5.0 */ public void setExtraParameters(Collection params) { - this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ + this.clientRequestedV2 = params.contains(VERSION_2_REQUEST); } /** @@ -1194,7 +1195,8 @@ private void fetchV2(PacketLineOut pckOut) throws IOException { new PacketLineOut(NullOutputStream.INSTANCE), accumulator); } else { - pckOut.writeString("acknowledgments\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_ACKNOWLEDGMENTS + '\n'); for (ObjectId id : req.getPeerHas()) { if (walk.getObjectReader().has(id)) { pckOut.writeString("ACK " + id.getName() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ @@ -1243,7 +1245,8 @@ private void fetchV2(PacketLineOut pckOut) throws IOException { if (!pckOut.isUsingSideband()) { // sendPack will write "packfile\n" for us if sideband-all is used. // But sideband-all is not used, so we have to write it ourselves. - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_PACKFILE + '\n'); } accumulator.timeNegotiating = Duration @@ -2327,7 +2330,8 @@ else if (ref.getName().startsWith(Constants.R_HEADS)) // for us if provided a PackfileUriConfig. In this case, we // are not providing a PackfileUriConfig, so we have to // write this line ourselves. - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_PACKFILE + '\n'); } } pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut);