From bcb5a431a51703c31ce9d9c1f933e7f00a463021 Mon Sep 17 00:00:00 2001 From: Christian Halstrick Date: Mon, 27 Jun 2016 16:00:44 +0200 Subject: [PATCH] Add built-in LFS smudge filter for local case Adds a JGit built-in implementation of the "git lfs smudge" filter. This filter should do the same as the one described in [1] besides that it only supports the local case when the lfs objects are already present in the media directory. Remote cases where download of LFS objects from an LFS server is needed will be done in a later commit. [1] https://github.com/github/git-lfs/blob/master/docs/man/git-lfs-smudge.1.ronn Change-Id: I8ff661d4edd3667ef7f86f3b4fa33e568eb4c8f4 --- org.eclipse.jgit.lfs/META-INF/MANIFEST.MF | 3 +- .../src/org/eclipse/jgit/lfs/LfsPointer.java | 48 ++++++- .../org/eclipse/jgit/lfs/SmudgeFilter.java | 125 ++++++++++++++++++ .../org/eclipse/jgit/lfs/lib/Constants.java | 7 + .../src/org/eclipse/jgit/pgm/Main.java | 2 + .../org/eclipse/jgit/api/AddCommandTest.java | 81 +++++++++++- .../eclipse/jgit/api/CheckoutCommandTest.java | 49 ++++--- .../src/org/eclipse/jgit/api/AddCommand.java | 5 + .../jgit/treewalk/WorkingTreeIterator.java | 17 +-- 9 files changed, 293 insertions(+), 44 deletions(-) create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index f1eeb512a..c8ba3e1bc 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -10,7 +10,8 @@ Export-Package: org.eclipse.jgit.lfs;version="4.6.0", org.eclipse.jgit.lfs.internal;version="4.6.0";x-friends:="org.eclipse.jgit.lfs.test", org.eclipse.jgit.lfs.lib;version="4.6.0" Bundle-RequiredExecutionEnvironment: JavaSE-1.7 -Import-Package: org.eclipse.jgit.attributes;version="[4.6.0,4.7.0)", +Import-Package: org.eclipse.jgit.annotations;version="[4.6.0,4.7.0)";resolution:=optional, + org.eclipse.jgit.attributes;version="[4.6.0,4.7.0)", org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)", org.eclipse.jgit.lib;version="[4.6.0,4.7.0)", org.eclipse.jgit.nls;version="[4.6.0,4.7.0)", diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java index 2521e6316..e43cb2563 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -42,9 +42,14 @@ */ package org.eclipse.jgit.lfs; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintStream; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.lfs.lib.Constants; import org.eclipse.jgit.lfs.lib.LongObjectId; @@ -113,9 +118,50 @@ public void encode(OutputStream out) { } } + /** + * Try to parse the data provided by an InputStream to the format defined by + * {@link #VERSION} + * + * @param in + * the {@link InputStream} from where to read the data + * @return an {@link LfsPointer} or null if the stream was not + * parseable as LfsPointer + * @throws IOException + */ + @Nullable + public static LfsPointer parseLfsPointer(InputStream in) + throws IOException { + boolean versionLine = false; + LongObjectId id = null; + long sz = -1; + + try (BufferedReader br = new BufferedReader( + new InputStreamReader(in))) { + for (String s = br.readLine(); s != null; s = br.readLine()) { + if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ + continue; + } else if (s.startsWith("version") && s.length() > 8 //$NON-NLS-1$ + && s.substring(8).trim().equals(VERSION)) { + versionLine = true; + } else if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ + id = LongObjectId.fromString(s.substring(11).trim()); + } else if (s.startsWith("size") && s.length() > 5) { //$NON-NLS-1$ + sz = Long.parseLong(s.substring(5).trim()); + } else { + return null; + } + } + if (versionLine && id != null && sz > -1) { + return new LfsPointer(id, sz); + } + } + return null; + } + @Override public String toString() { return "LfsPointer: oid=" + LongObjectId.toString(oid) + ", size=" //$NON-NLS-1$ //$NON-NLS-2$ + size; } -} \ No newline at end of file +} + diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java new file mode 100644 index 000000000..233247779 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2016, Christian Halstrick + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandFactory; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.Repository; + +/** + * Built-in LFS smudge filter + * + * When content is read from git's object-database and written to the filesystem + * and this filter is configured for that content, then this filter will replace + * the content of LFS pointer files with the original content. This happens e.g. + * when a checkout needs to update a working tree file which is under LFS + * control. This implementation expects that the origin content is already + * available in the .git/lfs/objects folder. This implementation will not + * contact any LFS servers in order to get the missing content. + * + * @since 4.6 + */ +public class SmudgeFilter extends FilterCommand { + /** + * The factory is responsible for creating instances of {@link SmudgeFilter} + */ + public final static FilterCommandFactory FACTORY = new FilterCommandFactory() { + @Override + public FilterCommand create(Repository db, InputStream in, + OutputStream out) throws IOException { + return new SmudgeFilter(db, in, out); + } + }; + + /** + * Registers this filter in JGit by calling + */ + public final static void register() { + FilterCommandRegistry.register( + org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + "lfs/smudge", //$NON-NLS-1$ + FACTORY); + } + + private Lfs lfs; + + /** + * @param db + * @param in + * @param out + * @throws IOException + */ + public SmudgeFilter(Repository db, InputStream in, OutputStream out) + throws IOException { + super(in, out); + lfs = new Lfs(db.getDirectory().toPath().resolve(Constants.LFS)); + LfsPointer res = LfsPointer.parseLfsPointer(in); + if (res != null) { + Path mediaFile = lfs.getMediaFile(res.getOid()); + if (Files.exists(mediaFile)) { + this.in = Files.newInputStream(mediaFile); + } + } + } + + @Override + public int run() throws IOException { + int b; + if (in != null) { + while ((b = in.read()) != -1) { + out.write(b); + } + in.close(); + } + out.close(); + return -1; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java index 269cbc3a8..20e87ae0b 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -55,6 +55,13 @@ **/ @SuppressWarnings("nls") public final class Constants { + /** + * lfs folder + * + * @since 4.6 + */ + public static final String LFS = "lfs"; + /** * Hash function used natively by Git LFS extension for large objects. * diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java index 654573b2a..3ddee36e3 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java @@ -58,6 +58,7 @@ import org.eclipse.jgit.awtui.AwtCredentialsProvider; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.lfs.CleanFilter; +import org.eclipse.jgit.lfs.SmudgeFilter; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryBuilder; import org.eclipse.jgit.pgm.internal.CLIText; @@ -99,6 +100,7 @@ public class Main { public Main() { HttpTransport.setConnectionFactory(new HttpClientConnectionFactory()); CleanFilter.register(); + SmudgeFilter.register(); } /** diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index a3cae4b05..5ad73f17b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -57,13 +57,21 @@ import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.NoFilepatternException; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lfs.CleanFilter; -import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.lfs.SmudgeFilter; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; @@ -79,12 +87,13 @@ @RunWith(Theories.class) public class AddCommandTest extends RepositoryTestCase { @DataPoints - public static boolean[] smudge = { true, false }; + public static boolean[] sleepBeforeAddOptions = { true, false }; @Override public void setUp() throws Exception { CleanFilter.register(); + SmudgeFilter.register(); super.setUp(); } @@ -149,7 +158,7 @@ public void testCleanFilter() throws IOException, GitAPIException { } @Theory - public void testBuiltinFilter(boolean doSmudge) + public void testBuiltinFilters(boolean sleepBeforeAdd) throws IOException, GitAPIException, InterruptedException { writeTrashFile(".gitattributes", "*.txt filter=lfs"); @@ -160,7 +169,7 @@ public void testBuiltinFilter(boolean doSmudge) File f = writeTrashFile("src/a.txt", "foo\n"); try (Git git = new Git(db)) { - if (!doSmudge) { + if (!sleepBeforeAdd) { fsTick(f); } git.add().addFilepattern(".gitattributes").call(); @@ -172,7 +181,7 @@ public void testBuiltinFilter(boolean doSmudge) config.setBoolean("filter", "lfs", "useJGitBuiltin", true); config.save(); - if (!doSmudge) { + if (!sleepBeforeAdd) { fsTick(f); } git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp") @@ -185,7 +194,67 @@ public void testBuiltinFilter(boolean doSmudge) RevCommit c1 = git.commit().setMessage("c1").call(); assertTrue(git.status().call().isClean()); f = writeTrashFile("src/a.txt", "foobar\n"); - if (!doSmudge) { + if (!sleepBeforeAdd) { + fsTick(f); + } + git.add().addFilepattern("src/a.txt").call(); + git.commit().setMessage("c2").call(); + assertTrue(git.status().call().isClean()); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f\nsize 7\n]", + indexState(CONTENT)); + assertEquals("foobar\n", read("src/a.txt")); + git.checkout().setName(c1.getName()).call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]", + indexState(CONTENT)); + assertEquals( + "foo\n", read("src/a.txt")); + } + } + + @Theory + public void testBuiltinCleanFilter(boolean sleepBeforeAdd) + throws IOException, GitAPIException, InterruptedException { + writeTrashFile(".gitattributes", "*.txt filter=lfs"); + writeTrashFile("src/a.tmp", "foo"); + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + File script = writeTempFile("sed s/o/e/g"); + File f = writeTrashFile("src/a.txt", "foo\n"); + + // unregister the smudge filter. Only clean filter should be builtin + FilterCommandRegistry.unregister( + org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + "lfs/smudge"); + + try (Git git = new Git(db)) { + if (!sleepBeforeAdd) { + fsTick(f); + } + git.add().addFilepattern(".gitattributes").call(); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "lfs", "clean", + "sh " + slashify(script.getPath())); + config.setString("filter", "lfs", "smudge", + "sh " + slashify(script.getPath())); + config.setBoolean("filter", "lfs", "useJGitBuiltin", true); + config.save(); + + if (!sleepBeforeAdd) { + fsTick(f); + } + git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp") + .addFilepattern(".gitattributes").call(); + + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]", + indexState(CONTENT)); + + RevCommit c1 = git.commit().setMessage("c1").call(); + assertTrue(git.status().call().isClean()); + f = writeTrashFile("src/a.txt", "foobar\n"); + if (!sleepBeforeAdd) { fsTick(f); } git.add().addFilepattern("src/a.txt").call(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java index 0bb6610a2..998b5fbfc 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java @@ -74,6 +74,8 @@ import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lfs.CleanFilter; +import org.eclipse.jgit.lfs.SmudgeFilter; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; @@ -100,6 +102,8 @@ public class CheckoutCommandTest extends RepositoryTestCase { @Override @Before public void setUp() throws Exception { + CleanFilter.register(); + SmudgeFilter.register(); super.setUp(); git = new Git(db); // commit something @@ -563,11 +567,11 @@ public void testCheckoutAutoCrlfTrue() throws Exception { public void testSmudgeFilter_modifyExisting() throws IOException, GitAPIException { File script = writeTempFile("sed s/o/e/g"); StoredConfig config = git.getRepository().getConfig(); - config.setString("filter", "tstFilter", "smudge", + config.setString("filter", "lfs", "smudge", "sh " + slashify(script.getPath())); config.save(); - writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); git.add().addFilepattern(".gitattributes").call(); git.commit().setMessage("add filter").call(); @@ -589,7 +593,7 @@ public void testSmudgeFilter_modifyExisting() throws IOException, GitAPIExceptio git.checkout().setName(content2.getName()).call(); assertEquals( - "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + "[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", indexState(CONTENT)); assertEquals(Sets.of("src/a.txt"), git.status().call().getModified()); assertEquals("foo", read("src/a.tmp")); @@ -601,7 +605,7 @@ public void testSmudgeFilter_createNew() throws IOException, GitAPIException { File script = writeTempFile("sed s/o/e/g"); StoredConfig config = git.getRepository().getConfig(); - config.setString("filter", "tstFilter", "smudge", + config.setString("filter", "lfs", "smudge", "sh " + slashify(script.getPath())); config.save(); @@ -609,7 +613,7 @@ public void testSmudgeFilter_createNew() git.add().addFilepattern("foo").call(); RevCommit initial = git.commit().setMessage("initial").call(); - writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); git.add().addFilepattern(".gitattributes").call(); git.commit().setMessage("add filter").call(); @@ -625,7 +629,7 @@ public void testSmudgeFilter_createNew() git.checkout().setName(content.getName()).call(); assertEquals( - "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + "[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", indexState(CONTENT)); assertEquals("foo", read("src/a.tmp")); assertEquals("fee\n", read("src/a.txt")); @@ -636,7 +640,7 @@ public void testSmudgeFilter_deleteFileAndRestoreFromCommit() throws IOException, GitAPIException { File script = writeTempFile("sed s/o/e/g"); StoredConfig config = git.getRepository().getConfig(); - config.setString("filter", "tstFilter", "smudge", + config.setString("filter", "lfs", "smudge", "sh " + slashify(script.getPath())); config.save(); @@ -644,7 +648,7 @@ public void testSmudgeFilter_deleteFileAndRestoreFromCommit() git.add().addFilepattern("foo").call(); git.commit().setMessage("initial").call(); - writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); git.add().addFilepattern(".gitattributes").call(); git.commit().setMessage("add filter").call(); @@ -661,7 +665,7 @@ public void testSmudgeFilter_deleteFileAndRestoreFromCommit() .call(); assertEquals( - "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + "[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", indexState(CONTENT)); assertEquals("foo", read("src/a.tmp")); assertEquals("fee\n", read("src/a.txt")); @@ -672,7 +676,7 @@ public void testSmudgeFilter_deleteFileAndRestoreFromIndex() throws IOException, GitAPIException { File script = writeTempFile("sed s/o/e/g"); StoredConfig config = git.getRepository().getConfig(); - config.setString("filter", "tstFilter", "smudge", + config.setString("filter", "lfs", "smudge", "sh " + slashify(script.getPath())); config.save(); @@ -680,7 +684,7 @@ public void testSmudgeFilter_deleteFileAndRestoreFromIndex() git.add().addFilepattern("foo").call(); git.commit().setMessage("initial").call(); - writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); git.add().addFilepattern(".gitattributes").call(); git.commit().setMessage("add filter").call(); @@ -696,7 +700,7 @@ public void testSmudgeFilter_deleteFileAndRestoreFromIndex() git.checkout().addPath("src/a.txt").call(); assertEquals( - "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + "[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", indexState(CONTENT)); assertEquals("foo", read("src/a.tmp")); assertEquals("fee\n", read("src/a.txt")); @@ -707,7 +711,7 @@ public void testSmudgeFilter_deleteFileAndCreateBranchAndRestoreFromCommit() throws IOException, GitAPIException { File script = writeTempFile("sed s/o/e/g"); StoredConfig config = git.getRepository().getConfig(); - config.setString("filter", "tstFilter", "smudge", + config.setString("filter", "lfs", "smudge", "sh " + slashify(script.getPath())); config.save(); @@ -715,7 +719,7 @@ public void testSmudgeFilter_deleteFileAndCreateBranchAndRestoreFromCommit() git.add().addFilepattern("foo").call(); git.commit().setMessage("initial").call(); - writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); git.add().addFilepattern(".gitattributes").call(); git.commit().setMessage("add filter").call(); @@ -732,7 +736,7 @@ public void testSmudgeFilter_deleteFileAndCreateBranchAndRestoreFromCommit() .setStartPoint(content).addPath("src/a.txt").call(); assertEquals( - "[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", + "[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]", indexState(CONTENT)); assertEquals("foo", read("src/a.tmp")); assertEquals("fee\n", read("src/a.txt")); @@ -745,12 +749,13 @@ public void testSmudgeAndClean() throws Exception { try (Git git2 = new Git(db)) { StoredConfig config = git.getRepository().getConfig(); - config.setString("filter", "tstFilter", "smudge", + config.setString("filter", "lfs", "smudge", "sh " + slashify(smudge_filter.getPath())); - config.setString("filter", "tstFilter", "clean", + config.setString("filter", "lfs", "clean", "sh " + slashify(clean_filter.getPath())); + config.setBoolean("filter", "lfs", "useJGitBuiltin", true); config.save(); - writeTrashFile(".gitattributes", "filterTest.txt filter=tstFilter"); + writeTrashFile(".gitattributes", "filterTest.txt filter=lfs"); git2.add().addFilepattern(".gitattributes").call(); git2.commit().setMessage("add attributes").call(); @@ -758,7 +763,7 @@ public void testSmudgeAndClean() throws Exception { git2.add().addFilepattern("filterTest.txt").call(); RevCommit one = git2.commit().setMessage("add filterText.txt").call(); assertEquals( - "[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:hello world, @version\n]", + "[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:7bd5d32e5c494354aa4c2473a1306d0ce7b52cc3bffeb342c03cd517ef8cf8da\nsize 16\n]", indexState(CONTENT)); fsTick(writeTrashFile("filterTest.txt", "bon giorno world, V1\n")); @@ -767,20 +772,20 @@ public void testSmudgeAndClean() throws Exception { assertTrue(git2.status().call().isClean()); assertEquals( - "[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:bon giorno world, @version\n]", + "[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:087148cccf53b0049c56475c1595113c9da4b638997c3489af8ac7108d51ef13\nsize 21\n]", indexState(CONTENT)); git2.checkout().setName(one.getName()).call(); assertTrue(git2.status().call().isClean()); assertEquals( - "[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:hello world, @version\n]", + "[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:7bd5d32e5c494354aa4c2473a1306d0ce7b52cc3bffeb342c03cd517ef8cf8da\nsize 16\n]", indexState(CONTENT)); assertEquals("hello world, V1\n", read("filterTest.txt")); git2.checkout().setName(two.getName()).call(); assertTrue(git2.status().call().isClean()); assertEquals( - "[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:bon giorno world, @version\n]", + "[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:087148cccf53b0049c56475c1595113c9da4b638997c3489af8ac7108d51ef13\nsize 21\n]", indexState(CONTENT)); assertEquals("bon giorno world, V1\n", read("filterTest.txt")); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index 1f37833a4..16ec1463c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -224,6 +224,11 @@ public DirCache call() throws GitAPIException, NoFilepatternException { entry.setLength(f.getEntryLength()); entry.setLastModified(f.getEntryLastModified()); long len = f.getEntryContentLength(); + // We read and filter the content multiple times. + // f.getEntryContentLength() reads and filters the input and + // inserter.insert(...) does it again. That's because an + // ObjectInserter needs to know the length before it starts + // inserting. TODO: Fix this by using Buffers. try (InputStream in = f.openEntryStream()) { ObjectId id = inserter.insert(OBJ_BLOB, len, in); entry.setObjectId(id); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index be4baef05..0a9c9578d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -267,7 +267,6 @@ public byte[] idBuffer() { // If there is a matching DirCacheIterator, we can reuse // its idBuffer, but only if we appear to be clean against // the cached index information for the path. - // DirCacheIterator i = state.walk.getTree(state.dirCacheTree, DirCacheIterator.class); if (i != null) { @@ -397,15 +396,9 @@ && getEolStreamType(opType) == EolStreamType.DIRECT) { if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) { ByteBuffer rawbuf = IO.readWholeStream(is, (int) len); - byte[] raw = rawbuf.array(); - int n = rawbuf.limit(); - if (!isBinary(raw, n)) { - rawbuf = filterClean(raw, n, opType); - raw = rawbuf.array(); - n = rawbuf.limit(); - } - canonLen = n; - return new ByteArrayInputStream(raw, 0, n); + rawbuf = filterClean(rawbuf.array(), rawbuf.limit(), opType); + canonLen = rawbuf.limit(); + return new ByteArrayInputStream(rawbuf.array(), 0, (int) canonLen); } if (getCleanFilterCommand() == null && isBinary(e)) { @@ -433,10 +426,6 @@ private static void safeClose(final InputStream in) { } } - private static boolean isBinary(byte[] content, int sz) { - return RawText.isBinary(content, sz); - } - private static boolean isBinary(Entry entry) throws IOException { InputStream in = entry.openInputStream(); try {