From 0e187f14843f1e1c73bfe6e80e91fdf03ae96067 Mon Sep 17 00:00:00 2001 From: Dariusz Luksza Date: Thu, 22 Oct 2015 16:34:04 +0200 Subject: [PATCH] Add LfsPointerFilter TreeFilter Add new variation of TreeFilter in order to detect LFS pointer files in the repository. Additionally, update LfsPointer to support the legacy version URL [1] as described in [2], and to allow arbitrary fields in the pointer file. [1] https://hawser.github.com/spec/v1 [2] https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md Change-Id: I621eb058619fb1b78888a54c4b60bb110a722fc3 Signed-off-by: Dariusz Luksza Signed-off-by: David Pursehouse Signed-off-by: Matthias Sohn --- .../META-INF/MANIFEST.MF | 6 +- .../jgit/lfs/lib/LfsPointerFilterTest.java | 214 ++++++++++++++++++ org.eclipse.jgit.lfs/META-INF/MANIFEST.MF | 3 + .../src/org/eclipse/jgit/lfs/LfsPointer.java | 11 +- .../jgit/lfs/lib/LfsPointerFilter.java | 103 +++++++++ org.eclipse.jgit/META-INF/MANIFEST.MF | 6 +- 6 files changed, 338 insertions(+), 5 deletions(-) create mode 100644 org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LfsPointerFilterTest.java create mode 100644 org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java diff --git a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF index c0ec91537..b5418cdad 100644 --- a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF @@ -6,11 +6,15 @@ Bundle-Version: 4.7.0.qualifier Bundle-Vendor: %provider_name Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.eclipse.jgit.junit;version="[4.7.0,4.8.0)", +Import-Package: org.eclipse.jgit.internal.storage.dfs;version="[4.7.0,4.8.0)", + org.eclipse.jgit.junit;version="[4.7.0,4.8.0)", org.eclipse.jgit.lfs;version="[4.7.0,4.8.0)", org.eclipse.jgit.lfs.errors;version="[4.7.0,4.8.0)", org.eclipse.jgit.lfs.lib;version="[4.7.0,4.8.0)", org.eclipse.jgit.lib;version="[4.7.0,4.8.0)", + org.eclipse.jgit.revwalk;version="[4.7.0,4.8.0)", + org.eclipse.jgit.treewalk;version="[4.7.0,4.8.0)", + org.eclipse.jgit.treewalk.filter;version="[4.7.0,4.8.0)", org.eclipse.jgit.util;version="[4.7.0,4.8.0)", org.hamcrest.core;version="[1.1.0,2.0.0)", org.junit;version="[4.0.0,5.0.0)", diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LfsPointerFilterTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LfsPointerFilterTest.java new file mode 100644 index 000000000..a56f0df89 --- /dev/null +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LfsPointerFilterTest.java @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2015, Dariusz Luksza + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.lfs.lib; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; +import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.junit.Test; + +public class LfsPointerFilterTest { + + private static final int SIZE = 12345; + + private static final String OID = "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"; + + private static final String[] NOT_VALID_LFS_FILES = { "", // empty file + // simulate java file + "package org.eclipse.jgit;", + // invalid LFS pointer, no oid and version + "version https://hawser.github.com/spec/v1\n", + // invalid LFS pointer, no version + "version https://hawser.github.com/spec/v1\n" + + "oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n", + // invalid LFS pointer, no id + "version https://hawser.github.com/spec/v1\n" + "size 12345\n", + // invalid LFS pointer, wrong order of oid and size + "version https://hawser.github.com/spec/v1\n" + "size 12345\n" + + "oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" }; + + private static final String[] LFS_VERSION_DOMAINS = { + "hawser", "git-lfs" + }; + + private static final String[] VALID_LFS_FILES = { + // valid LFS pointer + "version https://%s.github.com/spec/v1\n" + + "oid sha256:" + OID + "\n" + + "size " + SIZE + "\n", + // valid LFS pointer with "custom" key + "version https://%s.github.com/spec/v1\n" + + "custom key with value\n" + + "oid sha256:" + OID + "\n" + + "size " + SIZE + "\n", + // valid LFS pointer with key with "." + "version https://%s.github.com/spec/v1\n" + + "oid sha256:" + OID + "\n" + + "r.key key with .\n" + + "size " + SIZE + "\n", + // valid LFS pointer with key with "-" + "version https://%s.github.com/spec/v1\n" + + "oid sha256:" + OID + "\n" + + "size " + SIZE + "\n" + + "valid-name another valid key\n" }; + + @Test + public void testRegularFilesInRepositoryRoot() throws Exception { + for (String file : NOT_VALID_LFS_FILES) { + assertLfs("file.bin", file).withRecursive(false).shouldBe(false); + } + } + + @Test + public void testNestedRegularFiles() throws Exception { + for (String file : NOT_VALID_LFS_FILES) { + assertLfs("a/file.bin", file).withRecursive(true).shouldBe(false); + } + } + + @Test + public void testValidPointersInRepositoryRoot() throws Exception { + for (String domain : LFS_VERSION_DOMAINS) { + for (String file : VALID_LFS_FILES) { + assertLfs("file.bin", String.format(file, domain)) + .withRecursive(true).shouldBe(true) + .check(); + } + } + } + + @Test + public void testValidNestedPointers() throws Exception { + for (String domain : LFS_VERSION_DOMAINS) { + for (String file : VALID_LFS_FILES) { + assertLfs("a/file.bin", String.format(file, domain)) + .withRecursive(true).shouldBe(true).check(); + } + } + } + + @Test + public void testValidNestedPointersWithoutRecurrence() throws Exception { + for (String domain : LFS_VERSION_DOMAINS) { + for (String file : VALID_LFS_FILES) { + assertLfs("file.bin", String.format(file, domain)) + .withRecursive(false).shouldBe(true).check(); + assertLfs("a/file.bin", String.format(file, domain)) + .withRecursive(false).shouldBe(false).check(); + } + } + } + + private static LfsTreeWalk assertLfs(String path, String content) { + return new LfsTreeWalk(path, content); + } + + private static class LfsTreeWalk { + private final String path; + + private final String content; + + private boolean state; + + private boolean recursive; + + private TestRepository tr; + + LfsTreeWalk(String path, String content) { + this.path = path; + this.content = content; + } + + LfsTreeWalk withRecursive(boolean shouldBeRecursive) { + this.recursive = shouldBeRecursive; + return this; + } + + LfsTreeWalk shouldBe(boolean shouldBeValid) { + this.state = shouldBeValid; + return this; + } + + void check() throws Exception { + tr = new TestRepository<>(new InMemoryRepository( + new DfsRepositoryDescription("test"))); + RevCommit commit = tr.branch("master").commit().add(path, content) + .message("initial commit").create(); + RevTree tree = parseCommit(commit); + LfsPointerFilter filter = new LfsPointerFilter(); + try (TreeWalk treeWalk = new TreeWalk(tr.getRepository())) { + treeWalk.addTree(tree); + treeWalk.setRecursive(recursive); + treeWalk.setFilter(filter); + + if (state) { + assertTrue(treeWalk.next()); + assertEquals(path, treeWalk.getPathString()); + assertNotNull(filter.getPointer()); + assertEquals(SIZE, filter.getPointer().getSize()); + assertEquals(OID, filter.getPointer().getOid().name()); + } else { + assertFalse(treeWalk.next()); + assertNull(filter.getPointer()); + } + } + } + + private RevTree parseCommit(RevCommit commit) throws Exception { + try (ObjectWalk ow = new ObjectWalk(tr.getRepository())) { + return ow.parseCommit(commit).getTree(); + } + } + } +} diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index 01bf5a84c..c8e3ec84b 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -12,7 +12,10 @@ Export-Package: org.eclipse.jgit.lfs;version="4.7.0", Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: org.eclipse.jgit.annotations;version="[4.7.0,4.8.0)";resolution:=optional, org.eclipse.jgit.attributes;version="[4.7.0,4.8.0)", + org.eclipse.jgit.errors;version="[4.7.0,4.8.0)", org.eclipse.jgit.internal.storage.file;version="[4.7.0,4.8.0)", org.eclipse.jgit.lib;version="[4.7.0,4.8.0)", org.eclipse.jgit.nls;version="[4.7.0,4.8.0)", + org.eclipse.jgit.treewalk;version="[4.7.0,4.8.0)", + org.eclipse.jgit.treewalk.filter;version="[4.7.0,4.8.0)", org.eclipse.jgit.util;version="[4.7.0,4.8.0)" diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java index bbea53567..0aee1aadd 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -68,6 +68,12 @@ public class LfsPointer { */ public static final String VERSION = "https://git-lfs.github.com/spec/v1"; //$NON-NLS-1$ + /** + * The version of the LfsPointer file format using legacy URL + * @since 4.7 + */ + public static final String VERSION_LEGACY = "https://hawser.github.com/spec/v1"; //$NON-NLS-1$ + /** * The name of the hash function as used in the pointer files. This will * evaluate to "sha256" @@ -150,14 +156,13 @@ public static LfsPointer parseLfsPointer(InputStream in) if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ continue; } else if (s.startsWith("version") && s.length() > 8 //$NON-NLS-1$ - && s.substring(8).trim().equals(VERSION)) { + && (s.substring(8).trim().equals(VERSION) || + s.substring(8).trim().equals(VERSION_LEGACY))) { versionLine = true; } else if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ id = LongObjectId.fromString(s.substring(11).trim()); } else if (s.startsWith("size") && s.length() > 5) { //$NON-NLS-1$ sz = Long.parseLong(s.substring(5).trim()); - } else { - return null; } } if (versionLine && id != null && sz > -1) { diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java new file mode 100644 index 000000000..6f672b88f --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2015, 2017, Dariusz Luksza + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.lfs.lib; + +import java.io.IOException; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lfs.LfsPointer; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.TreeFilter; + +/** + * Detects Large File pointers, as described in [1] in Git repository. + * + * [1] https://github.com/github/git-lfs/blob/master/docs/spec.md + * + * @since 4.7 + */ +public class LfsPointerFilter extends TreeFilter { + + private LfsPointer pointer; + + /** + * @return {@link LfsPointer} or {@code null} + */ + public LfsPointer getPointer() { + return pointer; + } + + @Override + public boolean include(TreeWalk walk) throws MissingObjectException, + IncorrectObjectTypeException, IOException { + pointer = null; + if (walk.isSubtree()) { + return walk.isRecursive(); + } + ObjectId objectId = walk.getObjectId(0); + ObjectLoader object = walk.getObjectReader().open(objectId); + if (object.getSize() > 1024) { + return false; + } + + try (ObjectStream stream = object.openStream()) { + pointer = LfsPointer.parseLfsPointer(stream); + return pointer != null; + } + } + + @Override + public boolean shouldBeRecursive() { + return false; + } + + @Override + public TreeFilter clone() { + return new LfsPointerFilter(); + } +} diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index 903b0118b..8b7862820 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -60,7 +60,11 @@ Export-Package: org.eclipse.jgit.annotations;version="4.7.0", org.eclipse.jgit.ignore.internal;version="4.7.0";x-friends:="org.eclipse.jgit.test", org.eclipse.jgit.internal;version="4.7.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test", org.eclipse.jgit.internal.ketch;version="4.7.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.dfs;version="4.7.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.server,org.eclipse.jgit.http.test", + org.eclipse.jgit.internal.storage.dfs;version="4.7.0"; + x-friends:="org.eclipse.jgit.test, + org.eclipse.jgit.http.server, + org.eclipse.jgit.http.test, + org.eclipse.jgit.lfs.test", org.eclipse.jgit.internal.storage.file;version="4.7.0"; x-friends:="org.eclipse.jgit.test, org.eclipse.jgit.junit,