From 45ee55d0d9537c77f025d6c30e434149154d8ec4 Mon Sep 17 00:00:00 2001 From: Christian Halstrick Date: Mon, 27 Jun 2016 16:00:09 +0200 Subject: [PATCH] Add built-in LFS clean filter Adds a JGit built-in implementation of the "git lfs clean" filter. This filter should do the same as the one described in [1]. But since this filter is written in Java and can be called by JGit without forking new processes it should be much faster [1] https://github.com/github/git-lfs/blob/master/docs/man/git-lfs-clean.1.ronn Change-Id: If60e387e97870245b4bd765eda6717eb84cffb1d --- org.eclipse.jgit.lfs/META-INF/MANIFEST.MF | 6 +- .../jgit/lfs/internal/LfsText.properties | 3 +- .../src/org/eclipse/jgit/lfs/CleanFilter.java | 176 ++++++++++++++++++ .../src/org/eclipse/jgit/lfs/Lfs.java | 124 ++++++++++++ .../src/org/eclipse/jgit/lfs/LfsPointer.java | 121 ++++++++++++ .../jgit/lfs/errors/CorruptMediaFile.java | 100 ++++++++++ .../eclipse/jgit/lfs/internal/LfsText.java | 1 + .../org/eclipse/jgit/lfs/lib/Constants.java | 8 +- org.eclipse.jgit.pgm/META-INF/MANIFEST.MF | 1 + .../src/org/eclipse/jgit/pgm/Main.java | 2 + org.eclipse.jgit.test/META-INF/MANIFEST.MF | 1 + .../org/eclipse/jgit/api/AddCommandTest.java | 80 +++++++- 12 files changed, 616 insertions(+), 7 deletions(-) create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index d7a2f3749..f1eeb512a 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -5,11 +5,13 @@ Bundle-SymbolicName: org.eclipse.jgit.lfs Bundle-Version: 4.6.0.qualifier Bundle-Localization: plugin Bundle-Vendor: %provider_name -Export-Package: org.eclipse.jgit.lfs.errors;version="4.6.0", +Export-Package: org.eclipse.jgit.lfs;version="4.6.0", + org.eclipse.jgit.lfs.errors;version="4.6.0", org.eclipse.jgit.lfs.internal;version="4.6.0";x-friends:="org.eclipse.jgit.lfs.test", org.eclipse.jgit.lfs.lib;version="4.6.0" Bundle-RequiredExecutionEnvironment: JavaSE-1.7 -Import-Package: org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)", +Import-Package: org.eclipse.jgit.attributes;version="[4.6.0,4.7.0)", + org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)", org.eclipse.jgit.lib;version="[4.6.0,4.7.0)", org.eclipse.jgit.nls;version="[4.6.0,4.7.0)", org.eclipse.jgit.util;version="[4.6.0,4.7.0)" diff --git a/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties b/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties index 7c3aea226..25c101f0b 100644 --- a/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties +++ b/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties @@ -1,7 +1,8 @@ incorrectLONG_OBJECT_ID_LENGTH=Incorrect LONG_OBJECT_ID_LENGTH. +inconsistentMediafileLength=mediafile {0} has unexpected length; expected {1} but found {2}. invalidLongId=Invalid id: {0} invalidLongIdLength=Invalid id length {0}; should be {1} requiredHashFunctionNotAvailable=Required hash function {0} not available. repositoryNotFound=Repository {0} not found repositoryReadOnly=Repository {0} is read-only -lfsUnavailable=LFS is not available for repository {0} \ No newline at end of file +lfsUnavailable=LFS is not available for repository {0} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java new file mode 100644 index 000000000..f7b55e579 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2016, Christian Halstrick + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.DigestOutputStream; + +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandFactory; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.lfs.errors.CorruptMediaFile; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.FileUtils; + +/** + * Built-in LFS clean filter + * + * When new content is about to be added to the git repository and this filter + * is configured for that content, then this filter will replace the original + * content with content of a so-called LFS pointer file. The pointer file + * content will then be added to the git repository. Additionally this filter + * writes the original content in a so-called 'media file' to '.git/lfs/objects/ + * /' + * + * @see Git + * LFS Specification + * @since 4.6 + */ +public class CleanFilter extends FilterCommand { + /** + * The factory is responsible for creating instances of {@link CleanFilter} + */ + public final static FilterCommandFactory FACTORY = new FilterCommandFactory() { + + @Override + public FilterCommand create(Repository db, InputStream in, + OutputStream out) throws IOException { + return new CleanFilter(db, in, out); + } + }; + + /** + * Registers this filter by calling + * {@link FilterCommandRegistry#register(String, FilterCommandFactory)} + */ + public final static void register() { + FilterCommandRegistry.register( + org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + "lfs/clean", //$NON-NLS-1$ + FACTORY); + } + + // The OutputStream to a temporary file which will be renamed to mediafile + // when the operation succeeds + private OutputStream tmpOut; + + // Used to compute the hash for the original content + private DigestOutputStream dOut; + + private Lfs lfsUtil; + + // the size of the original content + private long size; + + // a temporary file into which the original content is written. When no + // errors occur this file will be renamed to the mediafile + private Path tmpFile; + + /** + * @param db + * the repository + * @param in + * an {@link InputStream} providing the original content + * @param out + * the {@link OutputStream} into which the content of the pointer + * file should be written. That's the content which will be added + * to the git repository + * @throws IOException + * when the creation of the temporary file fails or when no + * {@link OutputStream} for this file can be created + */ + public CleanFilter(Repository db, InputStream in, OutputStream out) + throws IOException { + super(in, out); + lfsUtil = new Lfs(db.getDirectory().toPath().resolve("lfs")); //$NON-NLS-1$ + Files.createDirectories(lfsUtil.getLfsTmpDir()); + tmpFile = lfsUtil.createTmpFile(); + tmpOut = Files.newOutputStream(tmpFile, + StandardOpenOption.CREATE); + this.dOut = new DigestOutputStream( + tmpOut, + Constants.newMessageDigest()); + } + + public int run() throws IOException { + try { + int b = in.read(); + if (b != -1) { + dOut.write(b); + size++; + return 1; + } else { + dOut.close(); + tmpOut.close(); + LongObjectId loid = LongObjectId + .fromRaw(dOut.getMessageDigest().digest()); + Path mediaFile = lfsUtil.getMediaFile(loid); + if (Files.isRegularFile(mediaFile)) { + long fsSize = Files.size(mediaFile); + if (fsSize != size) { + throw new CorruptMediaFile(mediaFile, size, fsSize); + } + } else { + FileUtils.mkdirs(mediaFile.getParent().toFile(), true); + FileUtils.rename(tmpFile.toFile(), mediaFile.toFile()); + } + LfsPointer lfsPointer = new LfsPointer(loid, size); + lfsPointer.encode(out); + out.close(); + return -1; + } + } catch (IOException e) { + out.close(); + dOut.close(); + tmpOut.close(); + throw e; + } + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java new file mode 100644 index 000000000..f099c5a76 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2016, Christian Halstrick + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.lfs.lib.LongObjectId; + +/** + * Class which represents the lfs folder hierarchy inside a .git folder + * + * @since 4.6 + */ +public class Lfs { + private Path root; + + private Path objDir; + + private Path tmpDir; + + /** + * @param root + * the path to the LFS media directory. Will be "/.git/lfs" + */ + public Lfs(Path root) { + this.root = root; + } + + /** + * @return the path to the LFS directory + */ + public Path getLfsRoot() { + return root; + } + + /** + * @return the path to the temp directory used by LFS. Will be + * "/.git/lfs/tmp" + */ + public Path getLfsTmpDir() { + if (tmpDir == null) { + tmpDir = root.resolve("tmp"); //$NON-NLS-1$ + } + return tmpDir; + } + + /** + * @return the path to the object directory used by LFS. Will be + * "/.git/lfs/objects" + */ + public Path getLfsObjDir() { + if (objDir == null) { + objDir = root.resolve("objects"); //$NON-NLS-1$ + } + return objDir; + } + + /** + * @param id + * the id of the mediafile + * @return the file which stores the original content. This will be files + * underneath + * "/.git/lfs/objects//" + */ + public Path getMediaFile(LongObjectId id) { + String idStr = LongObjectId.toString(id); + return getLfsObjDir().resolve(idStr.substring(0, 2)) + .resolve(idStr.substring(2)); + } + + /** + * Create a new temp file in the LFS directory + * + * @return a new temporary file in the LFS directory + * @throws IOException + * when the temp file could not be created + */ + public Path createTmpFile() throws IOException { + return Files.createTempFile(getLfsTmpDir(), null, null); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java new file mode 100644 index 000000000..2521e6316 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2016, Christian Halstrick + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.OutputStream; +import java.io.PrintStream; + +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; + +/** + * Represents an LFS pointer file + * + * @since 4.6 + */ +public class LfsPointer { + /** + * The version of the LfsPointer file format + */ + public static final String VERSION = "https://git-lfs.github.com/spec/v1"; //$NON-NLS-1$ + + /** + * The name of the hash function as used in the pointer files. This will + * evaluate to "sha256" + */ + public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION + .toLowerCase().replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$ + + private LongObjectId oid; + + private long size; + + /** + * @param oid + * the id of the content + * @param size + * the size of the content + */ + public LfsPointer(LongObjectId oid, long size) { + this.oid = oid; + this.size = size; + } + + /** + * @return the id of the content + */ + public LongObjectId getOid() { + return oid; + } + + /** + * @return the size of the content + */ + public long getSize() { + return size; + } + + /** + * Encode this object into the LFS format defined by {@link #VERSION} + * + * @param out + * the {@link OutputStream} into which the encoded data should be + * written + */ + public void encode(OutputStream out) { + try (PrintStream ps = new PrintStream(out)) { + ps.print("version "); //$NON-NLS-1$ + ps.println(VERSION); + ps.print("oid " + HASH_FUNCTION_NAME + ":"); //$NON-NLS-1$ //$NON-NLS-2$ + ps.println(LongObjectId.toString(oid)); + ps.print("size "); //$NON-NLS-1$ + ps.println(size); + } + } + + @Override + public String toString() { + return "LfsPointer: oid=" + LongObjectId.toString(oid) + ", size=" //$NON-NLS-1$ //$NON-NLS-2$ + + size; + } +} \ No newline at end of file diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java new file mode 100644 index 000000000..f2b51c044 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2016, Christian Halstrick + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs.errors; + +import java.io.IOException; +import java.nio.file.Path; +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when a LFS mediafile is found which doesn't have the expected size + * + * @since 4.6 + */ +public class CorruptMediaFile extends IOException { + private static final long serialVersionUID = 1L; + + private Path mediaFile; + + private long expectedSize; + + private long size; + + /** + * @param mediaFile + * @param expectedSize + * @param size + */ + @SuppressWarnings("boxing") + public CorruptMediaFile(Path mediaFile, long expectedSize, + long size) { + super(MessageFormat.format(LfsText.get().inconsistentMediafileLength, + mediaFile, expectedSize, size)); + this.mediaFile = mediaFile; + this.expectedSize = expectedSize; + this.size = size; + } + + /** + * @return the media file which seems to be corrupt + */ + public Path getMediaFile() { + return mediaFile; + } + + /** + * @return the expected size of the media file + */ + public long getExpectedSize() { + return expectedSize; + } + + /** + * @return the actual size of the media file in the file system + */ + public long getSize() { + return size; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java index 365eaa172..0aad5c9da 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java @@ -58,6 +58,7 @@ public static LfsText get() { } // @formatter:off + /***/ public String inconsistentMediafileLength; /***/ public String incorrectLONG_OBJECT_ID_LENGTH; /***/ public String invalidLongId; /***/ public String invalidLongIdLength; diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java index d2464126c..269cbc3a8 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -55,8 +55,12 @@ **/ @SuppressWarnings("nls") public final class Constants { - /** Hash function used natively by Git LFS extension for large objects. */ - private static final String LONG_HASH_FUNCTION = "SHA-256"; + /** + * Hash function used natively by Git LFS extension for large objects. + * + * @since 4.6 + */ + public static final String LONG_HASH_FUNCTION = "SHA-256"; /** * A Git LFS large object hash is 256 bits, i.e. 32 bytes. diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF index 3366d378c..7c2462b80 100644 --- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF @@ -39,6 +39,7 @@ Import-Package: javax.servlet;version="[3.1.0,4.0.0)", org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)", org.eclipse.jgit.internal.storage.pack;version="[4.6.0,4.7.0)", org.eclipse.jgit.internal.storage.reftree;version="[4.6.0,4.7.0)", + org.eclipse.jgit.lfs;version="[4.6.0,4.7.0)", org.eclipse.jgit.lfs.lib;version="[4.6.0,4.7.0)", org.eclipse.jgit.lfs.server;version="[4.6.0,4.7.0)", org.eclipse.jgit.lfs.server.fs;version="[4.6.0,4.7.0)", diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java index d701f22c3..654573b2a 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java @@ -57,6 +57,7 @@ import org.eclipse.jgit.awtui.AwtAuthenticator; import org.eclipse.jgit.awtui.AwtCredentialsProvider; import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lfs.CleanFilter; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryBuilder; import org.eclipse.jgit.pgm.internal.CLIText; @@ -97,6 +98,7 @@ public class Main { */ public Main() { HttpTransport.setConnectionFactory(new HttpClientConnectionFactory()); + CleanFilter.register(); } /** diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index c33603715..04bb247be 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -28,6 +28,7 @@ Import-Package: com.googlecode.javaewah;version="[0.7.9,0.8.0)", org.eclipse.jgit.internal.storage.pack;version="[4.6.0,4.7.0)", org.eclipse.jgit.internal.storage.reftree;version="[4.6.0,4.7.0)", org.eclipse.jgit.junit;version="[4.6.0,4.7.0)", + org.eclipse.jgit.lfs;version="[4.6.0,4.7.0)", org.eclipse.jgit.lib;version="[4.6.0,4.7.0)", org.eclipse.jgit.merge;version="[4.6.0,4.7.0)", org.eclipse.jgit.nls;version="[4.6.0,4.7.0)", diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index 42601aaa9..a3cae4b05 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -62,6 +62,7 @@ import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lfs.CleanFilter; import org.eclipse.jgit.lib.*; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; @@ -70,8 +71,22 @@ import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.junit.Test; +import org.junit.experimental.theories.DataPoints; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; +@RunWith(Theories.class) public class AddCommandTest extends RepositoryTestCase { + @DataPoints + public static boolean[] smudge = { true, false }; + + + @Override + public void setUp() throws Exception { + CleanFilter.register(); + super.setUp(); + } @Test public void testAddNothing() throws GitAPIException { @@ -110,8 +125,7 @@ public void testAddExistingSingleFile() throws IOException, GitAPIException { } @Test - public void testCleanFilter() throws IOException, - GitAPIException { + public void testCleanFilter() throws IOException, GitAPIException { writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); writeTrashFile("src/a.tmp", "foo"); // Caution: we need a trailing '\n' since sed on mac always appends @@ -134,6 +148,68 @@ public void testCleanFilter() throws IOException, } } + @Theory + public void testBuiltinFilter(boolean doSmudge) + throws IOException, + GitAPIException, InterruptedException { + writeTrashFile(".gitattributes", "*.txt filter=lfs"); + writeTrashFile("src/a.tmp", "foo"); + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + File script = writeTempFile("sed s/o/e/g"); + File f = writeTrashFile("src/a.txt", "foo\n"); + + try (Git git = new Git(db)) { + if (!doSmudge) { + fsTick(f); + } + git.add().addFilepattern(".gitattributes").call(); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "lfs", "clean", + "sh " + slashify(script.getPath())); + config.setString("filter", "lfs", "smudge", + "sh " + slashify(script.getPath())); + config.setBoolean("filter", "lfs", "useJGitBuiltin", true); + config.save(); + + if (!doSmudge) { + fsTick(f); + } + git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp") + .addFilepattern(".gitattributes").call(); + + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]", + indexState(CONTENT)); + + RevCommit c1 = git.commit().setMessage("c1").call(); + assertTrue(git.status().call().isClean()); + f = writeTrashFile("src/a.txt", "foobar\n"); + if (!doSmudge) { + fsTick(f); + } + git.add().addFilepattern("src/a.txt").call(); + git.commit().setMessage("c2").call(); + assertTrue(git.status().call().isClean()); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f\nsize 7\n]", + indexState(CONTENT)); + assertEquals("foobar\n", read("src/a.txt")); + git.checkout().setName(c1.getName()).call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]", + indexState(CONTENT)); + // due to lfs clean filter but dummy smudge filter we expect strange + // content. The smudge filter converts from real content to pointer + // file content (starting with "version ") but the smudge filter + // replaces 'o' by 'e' which results in a text starting with + // "versien " + assertEquals( + "versien https://git-lfs.github.cem/spec/v1\neid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n", + read("src/a.txt")); + } + } + @Test public void testAttributesWithTreeWalkFilter() throws IOException, GitAPIException {