From a229072fcd701f3d1f8835b36db95ec65ec2a6a2 Mon Sep 17 00:00:00 2001 From: Matthias Fromme Date: Mon, 3 Jan 2022 13:06:34 +0100 Subject: [PATCH] Support for "lfs.url" from ".lfsconfig" - New class LfsConfig to enrich repository configuration by settings from ".lfsconfig" file respecting configuration file precedence. - Adapted LfsConnectionFactory to use LfsConfig instead of directly using configuration from repository to calculate url of the lfs repository Bug: 578020 Change-Id: I156f4ec137c2e428136a2ca9b8a4011ecee2d915 --- .../META-INF/MANIFEST.MF | 1 + .../eclipse/jgit/lfs/LfsConfigGitTest.java | 309 ++++++++++++++++++ .../internal/LfsConnectionFactoryTest.java | 227 +++++++++++-- org.eclipse.jgit.lfs/META-INF/MANIFEST.MF | 4 +- .../jgit/lfs/internal/LfsText.properties | 3 +- .../eclipse/jgit/lfs/internal/LfsConfig.java | 200 ++++++++++++ .../lfs/internal/LfsConnectionFactory.java | 16 +- .../eclipse/jgit/lfs/internal/LfsText.java | 1 + .../org/eclipse/jgit/lfs/lib/Constants.java | 7 + 9 files changed, 737 insertions(+), 31 deletions(-) create mode 100644 org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsConfigGitTest.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java diff --git a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF index 7aafbe6d2..b29643dc7 100644 --- a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF @@ -18,6 +18,7 @@ Import-Package: org.eclipse.jgit.api;version="[6.1.0,6.2.0)", org.eclipse.jgit.lib;version="[6.1.0,6.2.0)", org.eclipse.jgit.revwalk;version="[6.1.0,6.2.0)", org.eclipse.jgit.transport;version="[6.1.0,6.2.0)", + org.eclipse.jgit.transport.http;version="[6.1.0,6.2.0)", org.eclipse.jgit.treewalk;version="[6.1.0,6.2.0)", org.eclipse.jgit.treewalk.filter;version="[6.1.0,6.2.0)", org.eclipse.jgit.util;version="[6.1.0,6.2.0)", diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsConfigGitTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsConfigGitTest.java new file mode 100644 index 000000000..98a0712e4 --- /dev/null +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsConfigGitTest.java @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2022, Matthias Fromme + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ResetCommand.ResetType; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lfs.internal.LfsConnectionFactory; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.util.HttpSupport; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test if the lfs config is used in the correct way during checkout. + * + * Two lfs-files are created, one that comes before .gitattributes and + * .lfsconfig in git order (".aaa.txt") and one that comes after ("zzz.txt"). + * + * During checkout/reset it is tested if the correct version of the lfs config + * is used. + * + * TODO: The current behavior seems a little bit strange/unintuitive. Some files + * are checked out before and some after the config files. This leads to the + * behavior, that during a single command the config changes. Since this seems + * to be the same way in native git, the behavior is accepted for now. + * + */ +public class LfsConfigGitTest extends RepositoryTestCase { + + private static final String SMUDGE_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE; + + private static final String LFS_SERVER_URI1 = "https://lfs.server1/test/uri"; + + private static final String EXPECTED_SERVER_URL1 = LFS_SERVER_URI1 + + Protocol.OBJECTS_LFS_ENDPOINT; + + private static final String LFS_SERVER_URI2 = "https://lfs.server2/test/uri"; + + private static final String EXPECTED_SERVER_URL2 = LFS_SERVER_URI2 + + Protocol.OBJECTS_LFS_ENDPOINT; + + private static final String LFS_SERVER_URI3 = "https://lfs.server3/test/uri"; + + private static final String EXPECTED_SERVER_URL3 = LFS_SERVER_URI3 + + Protocol.OBJECTS_LFS_ENDPOINT; + + private static final String FAKE_LFS_POINTER1 = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:6ce9fab52ee9a6c4c097def4e049c6acdeba44c99d26e83ba80adec1473c9b2d\n" + + "size 253952\n"; + + private static final String FAKE_LFS_POINTER2 = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:a4b711cd989863ae2038758a62672138347abbbae4076a7ad3a545fda7d08f82\n" + + "size 67072\n"; + + private static List checkoutURLs = new ArrayList<>(); + + static class SmudgeFilterMock extends FilterCommand { + public SmudgeFilterMock(Repository db, InputStream in, + OutputStream out) throws IOException { + super(in, out); + HttpConnection lfsServerConn = LfsConnectionFactory.getLfsConnection(db, + HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD); + checkoutURLs.add(lfsServerConn.getURL().toString()); + } + + @Override + public int run() throws IOException { + // Stupid no impl + in.transferTo(out); + return -1; + } + } + + @BeforeClass + public static void installLfs() { + FilterCommandRegistry.register(SMUDGE_NAME, SmudgeFilterMock::new); + } + + @AfterClass + public static void removeLfs() { + FilterCommandRegistry.unregister(SMUDGE_NAME); + } + + private Git git; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + git = new Git(db); + // commit something + writeTrashFile("Test.txt", "Hello world"); + git.add().addFilepattern("Test.txt").call(); + git.commit().setMessage("Initial commit").call(); + // prepare the config for LFS + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "lfs", "smudge", SMUDGE_NAME); + config.setString(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, "false"); + config.save(); + + fileBefore = null; + fileAfter = null; + configFile = null; + gitAttributesFile = null; + } + + File fileBefore; + + File fileAfter; + + File configFile; + + File gitAttributesFile; + + private void createLfsFiles(String lfsPointer) throws Exception { + /* + * FileNames ".aaa.txt" and "zzz.txt" seem to be sufficient to get the + * desired checkout order before and after ".lfsconfig", at least in a + * number of manual tries. Since the files to checkout are contained in + * a set (see DirCacheCheckout::doCheckout) the order cannot be + * guaranteed. + */ + + //File to be checked out before lfs config + String fileNameBefore = ".aaa.txt"; + fileBefore = writeTrashFile(fileNameBefore, lfsPointer); + git.add().addFilepattern(fileNameBefore).call(); + + // File to be checked out after lfs config + String fileNameAfter = "zzz.txt"; + fileAfter = writeTrashFile(fileNameAfter, lfsPointer); + git.add().addFilepattern(fileNameAfter).call(); + + git.commit().setMessage("Commit LFS Pointer files").call(); + } + + + private String addLfsConfigFiles(String lfsServerUrl) throws Exception { + // Add config files to the repo + String lfsConfig1 = createLfsConfig(lfsServerUrl); + git.add().addFilepattern(Constants.DOT_LFS_CONFIG).call(); + // Modify gitattributes on second call, to force checkout too. + if (gitAttributesFile == null) { + gitAttributesFile = writeTrashFile(".gitattributes", + "*.txt filter=lfs"); + } else { + gitAttributesFile = writeTrashFile(".gitattributes", + "*.txt filter=lfs\n"); + } + + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("Commit config files").call(); + return lfsConfig1; + } + + private String createLfsConfig(String lfsServerUrl) throws IOException { + String lfsConfig1 = "[lfs]\n url = " + lfsServerUrl; + configFile = writeTrashFile(Constants.DOT_LFS_CONFIG, lfsConfig1); + return lfsConfig1; + } + + @Test + public void checkoutLfsObjects_reset() throws Exception { + createLfsFiles(FAKE_LFS_POINTER1); + String lfsConfig1 = addLfsConfigFiles(LFS_SERVER_URI1); + + // Delete files to force action on reset + assertTrue(configFile.delete()); + assertTrue(fileBefore.delete()); + assertTrue(fileAfter.delete()); + + assertTrue(gitAttributesFile.delete()); + + // create config file with different url + createLfsConfig(LFS_SERVER_URI3); + + checkoutURLs.clear(); + git.reset().setMode(ResetType.HARD).call(); + + checkFile(configFile, lfsConfig1); + checkFile(fileBefore, FAKE_LFS_POINTER1); + checkFile(fileAfter, FAKE_LFS_POINTER1); + + assertEquals(2, checkoutURLs.size()); + // TODO: Should may be EXPECTED_SERVR_URL1 + assertEquals(EXPECTED_SERVER_URL3, checkoutURLs.get(0)); + assertEquals(EXPECTED_SERVER_URL1, checkoutURLs.get(1)); + } + + @Test + public void checkoutLfsObjects_BranchSwitch() throws Exception { + // Create a new branch "URL1" and add config files + git.checkout().setCreateBranch(true).setName("URL1").call(); + + createLfsFiles(FAKE_LFS_POINTER1); + String lfsConfig1 = addLfsConfigFiles(LFS_SERVER_URI1); + + // Create a second new branch "URL2" and add config files + git.checkout().setCreateBranch(true).setName("URL2").call(); + + createLfsFiles(FAKE_LFS_POINTER2); + String lfsConfig2 = addLfsConfigFiles(LFS_SERVER_URI2); + + checkFile(configFile, lfsConfig2); + checkFile(fileBefore, FAKE_LFS_POINTER2); + checkFile(fileAfter, FAKE_LFS_POINTER2); + + checkoutURLs.clear(); + git.checkout().setName("URL1").call(); + + checkFile(configFile, lfsConfig1); + checkFile(fileBefore, FAKE_LFS_POINTER1); + checkFile(fileAfter, FAKE_LFS_POINTER1); + + assertEquals(2, checkoutURLs.size()); + // TODO: Should may be EXPECTED_SERVR_URL1 + assertEquals(EXPECTED_SERVER_URL2, checkoutURLs.get(0)); + assertEquals(EXPECTED_SERVER_URL1, checkoutURLs.get(1)); + + checkoutURLs.clear(); + git.checkout().setName("URL2").call(); + + checkFile(configFile, lfsConfig2); + checkFile(fileBefore, FAKE_LFS_POINTER2); + checkFile(fileAfter, FAKE_LFS_POINTER2); + + assertEquals(2, checkoutURLs.size()); + // TODO: Should may be EXPECTED_SERVR_URL2 + assertEquals(EXPECTED_SERVER_URL1, checkoutURLs.get(0)); + assertEquals(EXPECTED_SERVER_URL2, checkoutURLs.get(1)); + } + + @Test + public void checkoutLfsObjects_BranchSwitch_ModifiedLocal() + throws Exception { + + // Create a new branch "URL1" and add config files + git.checkout().setCreateBranch(true).setName("URL1").call(); + + createLfsFiles(FAKE_LFS_POINTER1); + addLfsConfigFiles(LFS_SERVER_URI1); + + // Create a second new branch "URL2" and add config files + git.checkout().setCreateBranch(true).setName("URL2").call(); + + createLfsFiles(FAKE_LFS_POINTER2); + addLfsConfigFiles(LFS_SERVER_URI1); + + // create config file with different url + assertTrue(configFile.delete()); + String lfsConfig3 = createLfsConfig(LFS_SERVER_URI3); + + checkFile(configFile, lfsConfig3); + checkFile(fileBefore, FAKE_LFS_POINTER2); + checkFile(fileAfter, FAKE_LFS_POINTER2); + + checkoutURLs.clear(); + git.checkout().setName("URL1").call(); + + checkFile(fileBefore, FAKE_LFS_POINTER1); + checkFile(fileAfter, FAKE_LFS_POINTER1); + checkFile(configFile, lfsConfig3); + + assertEquals(2, checkoutURLs.size()); + + assertEquals(EXPECTED_SERVER_URL3, checkoutURLs.get(0)); + assertEquals(EXPECTED_SERVER_URL3, checkoutURLs.get(1)); + + checkoutURLs.clear(); + git.checkout().setName("URL2").call(); + + checkFile(fileBefore, FAKE_LFS_POINTER2); + checkFile(fileAfter, FAKE_LFS_POINTER2); + checkFile(configFile, lfsConfig3); + + assertEquals(2, checkoutURLs.size()); + assertEquals(EXPECTED_SERVER_URL3, checkoutURLs.get(0)); + assertEquals(EXPECTED_SERVER_URL3, checkoutURLs.get(1)); + } +} diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/internal/LfsConnectionFactoryTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/internal/LfsConnectionFactoryTest.java index c7bd48e12..badcb7d7e 100644 --- a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/internal/LfsConnectionFactoryTest.java +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/internal/LfsConnectionFactoryTest.java @@ -9,10 +9,15 @@ */ package org.eclipse.jgit.lfs.internal; +import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; -import java.util.TreeMap; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.RemoteAddCommand; @@ -27,6 +32,8 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.util.HttpSupport; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -42,6 +49,12 @@ public class LfsConnectionFactoryTest extends RepositoryTestCase { + Constants.ATTR_FILTER_DRIVER_PREFIX + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN; + private final static String LFS_SERVER_URL1 = "https://lfs.server1/test/uri"; + + private final static String LFS_SERVER_URL2 = "https://lfs.server2/test/uri"; + + private final static String ORIGIN_URL = "https://git.server/test/uri"; + private Git git; @BeforeClass @@ -61,26 +74,23 @@ public static void removeLfs() { public void setUp() throws Exception { super.setUp(); git = new Git(db); + + // Just to have a non empty repo + writeTrashFile("Test.txt", "Hello world from the LFS Factory Test"); + git.add().addFilepattern("Test.txt").call(); + git.commit().setMessage("Initial commit").call(); } @Test public void lfsUrlFromRemoteUrlWithDotGit() throws Exception { addRemoteUrl("https://localhost/repo.git"); - - String lfsUrl = LfsConnectionFactory.getLfsUrl(db, - Protocol.OPERATION_DOWNLOAD, - new TreeMap<>()); - assertEquals("https://localhost/repo.git/info/lfs", lfsUrl); + checkLfsUrl("https://localhost/repo.git/info/lfs"); } @Test public void lfsUrlFromRemoteUrlWithoutDotGit() throws Exception { addRemoteUrl("https://localhost/repo"); - - String lfsUrl = LfsConnectionFactory.getLfsUrl(db, - Protocol.OPERATION_DOWNLOAD, - new TreeMap<>()); - assertEquals("https://localhost/repo.git/info/lfs", lfsUrl); + checkLfsUrl("https://localhost/repo.git/info/lfs"); } @Test @@ -94,10 +104,7 @@ public void lfsUrlFromLocalConfig() throws Exception { "https://localhost/repo/lfs"); cfg.save(); - String lfsUrl = LfsConnectionFactory.getLfsUrl(db, - Protocol.OPERATION_DOWNLOAD, - new TreeMap<>()); - assertEquals("https://localhost/repo/lfs", lfsUrl); + checkLfsUrl("https://localhost/repo/lfs"); } @Test @@ -111,16 +118,136 @@ public void lfsUrlFromOriginConfig() throws Exception { "https://localhost/repo/lfs"); cfg.save(); - String lfsUrl = LfsConnectionFactory.getLfsUrl(db, - Protocol.OPERATION_DOWNLOAD, - new TreeMap<>()); - assertEquals("https://localhost/repo/lfs", lfsUrl); + checkLfsUrl("https://localhost/repo/lfs"); } @Test public void lfsUrlNotConfigured() throws Exception { - assertThrows(LfsConfigInvalidException.class, () -> LfsConnectionFactory - .getLfsUrl(db, Protocol.OPERATION_DOWNLOAD, new TreeMap<>())); + assertThrows(LfsConfigInvalidException.class, + () -> LfsConnectionFactory.getLfsConnection(db, + HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD)); + } + + @Test + public void checkGetLfsConnection_lfsurl_lfsconfigFromWorkingDir() + throws Exception { + writeLfsConfig(); + checkLfsUrl(LFS_SERVER_URL1); + } + + @Test + public void checkGetLfsConnection_lfsurl_lfsconfigFromIndex() + throws Exception { + writeLfsConfig(); + git.add().addFilepattern(Constants.DOT_LFS_CONFIG).call(); + deleteTrashFile(Constants.DOT_LFS_CONFIG); + checkLfsUrl(LFS_SERVER_URL1); + } + + @Test + public void checkGetLfsConnection_lfsurl_lfsconfigFromHEAD() + throws Exception { + writeLfsConfig(); + git.add().addFilepattern(Constants.DOT_LFS_CONFIG).call(); + git.commit().setMessage("Commit LFS Config").call(); + + /* + * reading .lfsconfig from HEAD seems only testable using a bare repo, + * since otherwise working tree or index are used + */ + File directory = createTempDirectory("testBareRepo"); + try (Repository bareRepoDb = Git.cloneRepository() + .setDirectory(directory) + .setURI(db.getDirectory().toURI().toString()).setBare(true) + .call().getRepository()) { + + checkLfsUrl(LFS_SERVER_URL1); + } + } + + @Test + public void checkGetLfsConnection_remote_lfsconfigFromWorkingDir() + throws Exception { + addRemoteUrl(ORIGIN_URL); + writeLfsConfig(LFS_SERVER_URL1, "lfs", DEFAULT_REMOTE_NAME, "url"); + checkLfsUrl(LFS_SERVER_URL1); + } + + /** + * Test the config file precedence. + * + * Checking only with the local repository config is sufficient since from + * that point the "normal" precedence is used. + * + * @throws Exception + */ + @Test + public void checkGetLfsConnection_ConfigFilePrecedence_lfsconfigFromWorkingDir() + throws Exception { + writeLfsConfig(); + checkLfsUrl(LFS_SERVER_URL1); + + StoredConfig config = git.getRepository().getConfig(); + config.setString(ConfigConstants.CONFIG_SECTION_LFS, null, + ConfigConstants.CONFIG_KEY_URL, LFS_SERVER_URL2); + config.save(); + + checkLfsUrl(LFS_SERVER_URL2); + } + + @Test + public void checkGetLfsConnection_InvalidLfsConfig_WorkingDir() + throws Exception { + writeInvalidLfsConfig(); + LfsConfigInvalidException actualException = assertThrows( + LfsConfigInvalidException.class, () -> { + LfsConnectionFactory.getLfsConnection(db, HttpSupport.METHOD_POST, + Protocol.OPERATION_DOWNLOAD); + }); + assertTrue(getStackTrace(actualException) + .contains("Invalid line in config file")); + } + + @Test + public void checkGetLfsConnection_InvalidLfsConfig_Index() + throws Exception { + writeInvalidLfsConfig(); + git.add().addFilepattern(Constants.DOT_LFS_CONFIG).call(); + deleteTrashFile(Constants.DOT_LFS_CONFIG); + LfsConfigInvalidException actualException = assertThrows( + LfsConfigInvalidException.class, () -> { + LfsConnectionFactory.getLfsConnection(db, HttpSupport.METHOD_POST, + Protocol.OPERATION_DOWNLOAD); + }); + assertTrue(getStackTrace(actualException) + .contains("Invalid line in config file")); + } + + @Test + public void checkGetLfsConnection_InvalidLfsConfig_HEAD() throws Exception { + writeInvalidLfsConfig(); + git.add().addFilepattern(Constants.DOT_LFS_CONFIG).call(); + git.commit().setMessage("Commit LFS Config").call(); + + /* + * reading .lfsconfig from HEAD seems only testable using a bare repo, + * since otherwise working tree or index are used + */ + File directory = createTempDirectory("testBareRepo"); + try (Repository bareRepoDb = Git.cloneRepository() + .setDirectory(directory) + .setURI(db.getDirectory().toURI().toString()).setBare(true) + .call().getRepository()) { + LfsConfigInvalidException actualException = assertThrows( + LfsConfigInvalidException.class, + () -> { + LfsConnectionFactory.getLfsConnection(db, + HttpSupport.METHOD_POST, + Protocol.OPERATION_DOWNLOAD); + }); + assertTrue(getStackTrace(actualException) + .contains("Invalid line in config file")); + } } private void addRemoteUrl(String remotUrl) throws Exception { @@ -129,4 +256,62 @@ private void addRemoteUrl(String remotUrl) throws Exception { add.setName(org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME); add.call(); } + + /** + * Returns the stack trace of the provided exception as string + * + * @param actualException + * @return The exception stack trace as string + */ + private String getStackTrace(Exception actualException) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + actualException.printStackTrace(pw); + return sw.toString(); + } + + private void writeLfsConfig() throws IOException { + writeLfsConfig(LFS_SERVER_URL1, "lfs", "url"); + } + + private void writeLfsConfig(String lfsUrl, String section, String name) + throws IOException { + writeLfsConfig(lfsUrl, section, null, name); + } + + /* + * Write simple lfs config with single entry. Do not use FileBasedConfig to + * avoid introducing new dependency (for now). + */ + private void writeLfsConfig(String lfsUrl, String section, + String subsection, String name) throws IOException { + StringBuilder config = new StringBuilder(); + config.append("["); + config.append(section); + if (subsection != null) { + config.append(" \""); + config.append(subsection); + config.append("\""); + } + config.append("]\n"); + config.append(" "); + config.append(name); + config.append(" = "); + config.append(lfsUrl); + writeTrashFile(Constants.DOT_LFS_CONFIG, config.toString()); + } + + private void writeInvalidLfsConfig() throws IOException { + writeTrashFile(Constants.DOT_LFS_CONFIG, + "{lfs]\n url = " + LFS_SERVER_URL1); + } + + private void checkLfsUrl(String lfsUrl) throws IOException { + HttpConnection lfsServerConn; + lfsServerConn = LfsConnectionFactory.getLfsConnection(db, + HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD); + + assertEquals(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT, + lfsServerConn.getURL().toString()); + } } diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index 536991896..47677f8c6 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -17,6 +17,7 @@ Import-Package: com.google.gson;version="[2.8.2,3.0.0)", org.eclipse.jgit.api.errors;version="[6.1.0,6.2.0)", org.eclipse.jgit.attributes;version="[6.1.0,6.2.0)", org.eclipse.jgit.diff;version="[6.1.0,6.2.0)", + org.eclipse.jgit.dircache;version="[6.1.0,6.2.0)", org.eclipse.jgit.errors;version="[6.1.0,6.2.0)", org.eclipse.jgit.hooks;version="[6.1.0,6.2.0)", org.eclipse.jgit.internal.storage.file;version="[6.1.0,6.2.0)", @@ -30,4 +31,5 @@ Import-Package: com.google.gson;version="[2.8.2,3.0.0)", org.eclipse.jgit.treewalk;version="[6.1.0,6.2.0)", org.eclipse.jgit.treewalk.filter;version="[6.1.0,6.2.0)", org.eclipse.jgit.util;version="[6.1.0,6.2.0)", - org.eclipse.jgit.util.io;version="[6.1.0,6.2.0)" + org.eclipse.jgit.util.io;version="[6.1.0,6.2.0)", + org.slf4j;version="[1.7.0,2.0.0)" diff --git a/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties b/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties index 0e00f146a..0e2023cbd 100644 --- a/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties +++ b/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties @@ -16,4 +16,5 @@ lfsNoDownloadUrl="Need to download object from LFS server but couldn't determine serverFailure=When trying to open a connection to {0} the server responded with an error code. rc={1} wrongAmoutOfDataReceived=While downloading data from the content server {0} {1} bytes have been received while {2} have been expected userConfigInvalid="User config file {0} invalid {1}" -missingLocalObject="Local Object {0} is missing" \ No newline at end of file +missingLocalObject="Local Object {0} is missing" +dotLfsConfigReadFailed=Reading .lfsconfig failed \ No newline at end of file diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java new file mode 100644 index 000000000..71d395ca8 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2022, Matthias Fromme + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import java.io.File; +import java.io.IOException; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.BlobBasedConfig; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.treewalk.TreeWalk; + +import static org.eclipse.jgit.lib.Constants.HEAD; + +/** + * Encapsulate access to the .lfsconfig. + * + * According to the document + * https://github.com/git-lfs/git-lfs/blob/main/docs/man/git-lfs-config.5.ronn + * the order to find the .lfsconfig file is: + * + * 
+ *   1. in the root of the working tree
+ *   2. in the index
+ *   3. in the HEAD, for bare repositories this is the only place
+ *      that is searched
+ *
+ * + * Values from the .lfsconfig are used only if not specified in another git + * config file to allow local override without modifiction of a committed file. + */ +public class LfsConfig { + private Repository db; + private Config delegate; + + /** + * Create a new instance of the LfsConfig. + * + * @param db + * the associated repo + * @throws IOException + */ + public LfsConfig(Repository db) throws IOException { + this.db = db; + delegate = this.load(); + } + + /** + * Read the .lfsconfig file from the repository + * + * @return The loaded lfs config or null if it does not exist + * + * @throws IOException + */ + private Config load() throws IOException { + Config result = null; + + if (!db.isBare()) { + result = loadFromWorkingTree(); + if (result == null) { + result = loadFromIndex(); + } + } + + if (result == null) { + result = loadFromHead(); + } + + if (result == null) { + result = emptyConfig(); + } + + return result; + } + + /** + * Try to read the lfs config from a file called .lfsconfig at the top level + * of the working tree. + * + * @return the config, or null + * @throws IOException + */ + @Nullable + private Config loadFromWorkingTree() + throws IOException { + File lfsConfig = db.getFS().resolve(db.getWorkTree(), + Constants.DOT_LFS_CONFIG); + if (lfsConfig.exists() && lfsConfig.isFile()) { + FileBasedConfig config = new FileBasedConfig(lfsConfig, db.getFS()); + try { + config.load(); + return config; + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + } + return null; + } + + /** + * Try to read the lfs config from an entry called .lfsconfig contained in + * the index. + * + * @return the config, or null if the entry does not exist + * @throws IOException + */ + @Nullable + private Config loadFromIndex() + throws IOException { + try { + DirCacheEntry entry = db.readDirCache() + .getEntry(Constants.DOT_LFS_CONFIG); + if (entry != null) { + return new BlobBasedConfig(null, db, entry.getObjectId()); + } + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + return null; + } + + /** + * Try to read the lfs config from an entry called .lfsconfig contained in + * the head revision. + * + * @return the config, or null if the file does not exist + * @throws IOException + */ + @Nullable + private Config loadFromHead() throws IOException { + try (RevWalk revWalk = new RevWalk(db)) { + ObjectId headCommitId = db.resolve(HEAD); + if (headCommitId == null) { + return null; + } + RevCommit commit = revWalk.parseCommit(headCommitId); + RevTree tree = commit.getTree(); + TreeWalk treewalk = TreeWalk.forPath(db, Constants.DOT_LFS_CONFIG, + tree); + if (treewalk != null) { + return new BlobBasedConfig(null, db, treewalk.getObjectId(0)); + } + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + return null; + } + + /** + * Create an empty config as fallback to avoid null pointer checks. + * + * @return an empty config + */ + private Config emptyConfig() { + return new Config(); + } + + /** + * Get string value or null if not found. + * + * First tries to find the value in the git config files. If not found tries + * to find data in .lfsconfig. + * + * @param section + * the section + * @param subsection + * the subsection for the value + * @param name + * the key name + * @return a String value from the config, null if not found + */ + public String getString(final String section, final String subsection, + final String name) { + String result = db.getConfig().getString(section, subsection, name); + if (result == null) { + result = delegate.getString(section, subsection, name); + } + return result; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java index 5a17d411c..12b688d15 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java @@ -45,7 +45,6 @@ * Provides means to get a valid LFS connection for a given repository. */ public class LfsConnectionFactory { - private static final int SSH_AUTH_TIMEOUT_SECONDS = 30; private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$ private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$ @@ -104,19 +103,19 @@ public static HttpConnection getLfsConnection(Repository db, String method, * additional headers that can be used to connect to LFS server * @return the URL for the LFS server. e.g. * "https://github.com/github/git-lfs.git/info/lfs" - * @throws LfsConfigInvalidException - * if the LFS config is invalid + * @throws IOException + * if the LFS config is invalid or cannot be accessed * @see * Server Discovery documentation */ - static String getLfsUrl(Repository db, String purpose, + private static String getLfsUrl(Repository db, String purpose, Map additionalHeaders) - throws LfsConfigInvalidException { - StoredConfig config = db.getConfig(); + throws IOException { + LfsConfig config = new LfsConfig(db); String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, - null, - ConfigConstants.CONFIG_KEY_URL); + null, ConfigConstants.CONFIG_KEY_URL); + Exception ex = null; if (lfsUrl == null) { String remoteUrl = null; @@ -124,6 +123,7 @@ static String getLfsUrl(Repository db, String purpose, lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, remote, ConfigConstants.CONFIG_KEY_URL); + // This could be done better (more precise logic), but according // to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs // generally only supports 'origin' in an integrated workflow. diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java index 1ca37a9f6..22813ff64 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java @@ -45,4 +45,5 @@ public static LfsText get() { /***/ public String wrongAmoutOfDataReceived; /***/ public String userConfigInvalid; /***/ public String missingLocalObject; + /***/ public String dotLfsConfigReadFailed; } diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java index 3212a6350..9b41ec31f 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -81,6 +81,13 @@ public final class Constants { */ public static final String ATTR_FILTER_DRIVER_PREFIX = "lfs/"; + /** + * Config file name for lfs specific configuration + * + * @since 6.1 + */ + public static final String DOT_LFS_CONFIG = ".lfsconfig"; + /** * Create a new digest function for objects. *