LFS: Dramatically improve checkout speed with SSH authentication
SSH Authentication is quite expensive (~120ms on localhost against Gerrit with LFS plugin). The SSH authentication typically also sends a validity time of the returned token, which allows to re-use it for a certain time, avoiding the expensive authentication on every download request. This improves checkout times by large factors depending on the LFS object amount/sizes. Also make sure that all instances of Gson used by LFS are configured in the same way. Change-Id: I422c94c37021b4322789b3829fa0185e25d683f2 Signed-off-by: Markus Duft <markus.duft@ssi-schaefer.com>
This commit is contained in:
Markus Duft
parent
c0bb992845
commit
ea2f7e93c7
|
|
@ -43,8 +43,8 @@
|
|||
package org.eclipse.jgit.lfs;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest;
|
||||
import static org.eclipse.jgit.lfs.Protocol.OPERATION_UPLOAD;
|
||||
import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest;
|
||||
import static org.eclipse.jgit.transport.http.HttpConnection.HTTP_OK;
|
||||
import static org.eclipse.jgit.util.HttpSupport.METHOD_POST;
|
||||
import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT;
|
||||
|
|
@ -123,6 +123,7 @@ public String call() throws IOException, AbortedByHookException {
|
|||
Map<String, LfsPointer> oid2ptr = requestBatchUpload(api, toPush);
|
||||
uploadContents(api, oid2ptr);
|
||||
return EMPTY;
|
||||
|
||||
}
|
||||
|
||||
private Set<LfsPointer> findObjectsToPush() throws IOException,
|
||||
|
|
@ -201,7 +202,7 @@ private Map<String, LfsPointer> requestBatchUpload(HttpConnection api,
|
|||
for (LfsPointer p : res) {
|
||||
oidStr2ptr.put(p.getOid().name(), p);
|
||||
}
|
||||
Gson gson = new Gson();
|
||||
Gson gson = Protocol.gson();
|
||||
api.getOutputStream().write(
|
||||
gson.toJson(toRequest(OPERATION_UPLOAD, res)).getBytes(UTF_8));
|
||||
int responseCode = api.getResponseCode();
|
||||
|
|
|
|||
|
|
@ -46,6 +46,10 @@
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.gson.FieldNamingPolicy;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
|
||||
/**
|
||||
* This interface describes the network protocol used between lfs client and lfs
|
||||
* server
|
||||
|
|
@ -97,6 +101,24 @@ class Action {
|
|||
public Map<String, String> header;
|
||||
}
|
||||
|
||||
/**
|
||||
* An action with an additional expiration timestamp
|
||||
*
|
||||
* @since 4.11
|
||||
*/
|
||||
class ExpiringAction extends Action {
|
||||
/**
|
||||
* Absolute date/time in format "yyyy-MM-dd'T'HH:mm:ss.SSSX"
|
||||
*/
|
||||
public String expiresAt;
|
||||
|
||||
/**
|
||||
* Validity time in milliseconds (preferred over expiresAt as specified:
|
||||
* https://github.com/git-lfs/git-lfs/blob/master/docs/api/authentication.md)
|
||||
*/
|
||||
public String expiresIn;
|
||||
}
|
||||
|
||||
/** Describes an error to be returned by the LFS batch API */
|
||||
class Error {
|
||||
public int code;
|
||||
|
|
@ -138,4 +160,17 @@ class Error {
|
|||
* Path to the LFS objects servlet.
|
||||
*/
|
||||
String OBJECTS_LFS_ENDPOINT = "/objects/batch"; //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* @return a {@link Gson} instance suitable for handling this
|
||||
* {@link Protocol}
|
||||
*
|
||||
* @since 4.11
|
||||
*/
|
||||
public static Gson gson() {
|
||||
return new GsonBuilder()
|
||||
.setFieldNamingPolicy(
|
||||
FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
|
||||
.disableHtmlEscaping().create();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ private Collection<Path> downloadLfsResource(Repository db,
|
|||
}
|
||||
HttpConnection lfsServerConn = LfsConnectionFactory.getLfsConnection(db,
|
||||
HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD);
|
||||
Gson gson = new Gson();
|
||||
Gson gson = Protocol.gson();
|
||||
lfsServerConn.getOutputStream()
|
||||
.write(gson
|
||||
.toJson(LfsConnectionFactory
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@
|
|||
import java.io.InputStreamReader;
|
||||
import java.net.ProxySelector;
|
||||
import java.net.URL;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.LinkedList;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
|
@ -75,8 +76,6 @@
|
|||
import org.eclipse.jgit.util.io.MessageWriter;
|
||||
import org.eclipse.jgit.util.io.StreamCopyThread;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
|
||||
/**
|
||||
* Provides means to get a valid LFS connection for a given repository.
|
||||
*/
|
||||
|
|
@ -84,6 +83,7 @@ public class LfsConnectionFactory {
|
|||
|
||||
private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
|
||||
private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
|
||||
private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
|
||||
|
||||
/**
|
||||
* Determine URL of LFS server by looking into config parameters lfs.url,
|
||||
|
|
@ -166,17 +166,9 @@ private static String discoverLfsUrl(Repository db, String purpose,
|
|||
Map<String, String> additionalHeaders, String remoteUrl) {
|
||||
try {
|
||||
URIish u = new URIish(remoteUrl);
|
||||
|
||||
if (SCHEME_SSH.equals(u.getScheme())) {
|
||||
// discover and authenticate; git-lfs does "ssh -p
|
||||
// <port> -- <host> git-lfs-authenticate <project>
|
||||
// <upload/download>"
|
||||
String json = runSshCommand(u.setPath(""), db.getFS(), //$NON-NLS-1$
|
||||
"git-lfs-authenticate " + extractProjectName(u) //$NON-NLS-1$
|
||||
+ " " + purpose); //$NON-NLS-1$
|
||||
|
||||
Protocol.Action action = new Gson().fromJson(json,
|
||||
Protocol.Action.class);
|
||||
Protocol.ExpiringAction action = getSshAuthentication(
|
||||
db, purpose, remoteUrl, u);
|
||||
additionalHeaders.putAll(action.header);
|
||||
return action.href;
|
||||
} else {
|
||||
|
|
@ -187,6 +179,34 @@ private static String discoverLfsUrl(Repository db, String purpose,
|
|||
}
|
||||
}
|
||||
|
||||
private static Protocol.ExpiringAction getSshAuthentication(
|
||||
Repository db, String purpose, String remoteUrl, URIish u)
|
||||
throws IOException {
|
||||
AuthCache cached = sshAuthCache.get(remoteUrl);
|
||||
Protocol.ExpiringAction action = null;
|
||||
if (cached != null && cached.validUntil > System.currentTimeMillis()) {
|
||||
action = cached.cachedAction;
|
||||
}
|
||||
|
||||
if (action == null) {
|
||||
// discover and authenticate; git-lfs does "ssh
|
||||
// -p <port> -- <host> git-lfs-authenticate
|
||||
// <project> <upload/download>"
|
||||
String json = runSshCommand(u.setPath(""), //$NON-NLS-1$
|
||||
db.getFS(),
|
||||
"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
|
||||
+ purpose);
|
||||
|
||||
action = Protocol.gson().fromJson(json,
|
||||
Protocol.ExpiringAction.class);
|
||||
|
||||
// cache the result as long as possible.
|
||||
AuthCache c = new AuthCache(action);
|
||||
sshAuthCache.put(remoteUrl, c);
|
||||
}
|
||||
return action;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a connection for the specified
|
||||
* {@link org.eclipse.jgit.lfs.Protocol.Action}.
|
||||
|
|
@ -291,4 +311,41 @@ public static Protocol.Request toRequest(String operation,
|
|||
return req;
|
||||
}
|
||||
|
||||
private static final class AuthCache {
|
||||
private static final long AUTH_CACHE_EAGER_TIMEOUT = 100;
|
||||
|
||||
private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
|
||||
"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* Creates a cache entry for an authentication response.
|
||||
* <p>
|
||||
* The timeout of the cache token is extracted from the given action. If
|
||||
* no timeout can be determined, the token will be used only once.
|
||||
*
|
||||
* @param action
|
||||
*/
|
||||
public AuthCache(Protocol.ExpiringAction action) {
|
||||
this.cachedAction = action;
|
||||
try {
|
||||
if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
|
||||
this.validUntil = System.currentTimeMillis()
|
||||
+ Long.parseLong(action.expiresIn);
|
||||
} else if (action.expiresAt != null
|
||||
&& !action.expiresAt.isEmpty()) {
|
||||
this.validUntil = ISO_FORMAT.parse(action.expiresAt)
|
||||
.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
|
||||
} else {
|
||||
this.validUntil = System.currentTimeMillis();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
this.validUntil = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
|
||||
long validUntil;
|
||||
|
||||
Protocol.ExpiringAction cachedAction;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue