Add LfsPointerFilter TreeFilter
Add new variation of TreeFilter in order to detect LFS pointer files in the repository. Additionally, update LfsPointer to support the legacy version URL [1] as described in [2], and to allow arbitrary fields in the pointer file. [1] https://hawser.github.com/spec/v1 [2] https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md Change-Id: I621eb058619fb1b78888a54c4b60bb110a722fc3 Signed-off-by: Dariusz Luksza <dariusz@luksza.org> Signed-off-by: David Pursehouse <david.pursehouse@gmail.com> Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
This commit is contained in:
parent
db77610256
commit
0e187f1484
|
@ -6,11 +6,15 @@ Bundle-Version: 4.7.0.qualifier
|
|||
Bundle-Vendor: %provider_name
|
||||
Bundle-Localization: plugin
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
|
||||
Import-Package: org.eclipse.jgit.junit;version="[4.7.0,4.8.0)",
|
||||
Import-Package: org.eclipse.jgit.internal.storage.dfs;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.junit;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.lfs;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.lfs.errors;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.lfs.lib;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.lib;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.revwalk;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.treewalk;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.treewalk.filter;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.util;version="[4.7.0,4.8.0)",
|
||||
org.hamcrest.core;version="[1.1.0,2.0.0)",
|
||||
org.junit;version="[4.0.0,5.0.0)",
|
||||
|
|
|
@ -0,0 +1,214 @@
|
|||
/*
|
||||
* Copyright (C) 2015, Dariusz Luksza <dariusz@luksza.org>
|
||||
* and other copyright owners as documented in the project's IP log.
|
||||
*
|
||||
* This program and the accompanying materials are made available
|
||||
* under the terms of the Eclipse Distribution License v1.0 which
|
||||
* accompanies this distribution, is reproduced below, and is
|
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or
|
||||
* without modification, are permitted provided that the following
|
||||
* conditions are met:
|
||||
*
|
||||
* - Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
*
|
||||
* - Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following
|
||||
* disclaimer in the documentation and/or other materials provided
|
||||
* with the distribution.
|
||||
*
|
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the
|
||||
* names of its contributors may be used to endorse or promote
|
||||
* products derived from this software without specific prior
|
||||
* written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
package org.eclipse.jgit.lfs.lib;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription;
|
||||
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository;
|
||||
import org.eclipse.jgit.junit.TestRepository;
|
||||
import org.eclipse.jgit.revwalk.ObjectWalk;
|
||||
import org.eclipse.jgit.revwalk.RevCommit;
|
||||
import org.eclipse.jgit.revwalk.RevTree;
|
||||
import org.eclipse.jgit.treewalk.TreeWalk;
|
||||
import org.junit.Test;
|
||||
|
||||
public class LfsPointerFilterTest {
|
||||
|
||||
private static final int SIZE = 12345;
|
||||
|
||||
private static final String OID = "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393";
|
||||
|
||||
private static final String[] NOT_VALID_LFS_FILES = { "", // empty file
|
||||
// simulate java file
|
||||
"package org.eclipse.jgit;",
|
||||
// invalid LFS pointer, no oid and version
|
||||
"version https://hawser.github.com/spec/v1\n",
|
||||
// invalid LFS pointer, no version
|
||||
"version https://hawser.github.com/spec/v1\n"
|
||||
+ "oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n",
|
||||
// invalid LFS pointer, no id
|
||||
"version https://hawser.github.com/spec/v1\n" + "size 12345\n",
|
||||
// invalid LFS pointer, wrong order of oid and size
|
||||
"version https://hawser.github.com/spec/v1\n" + "size 12345\n"
|
||||
+ "oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" };
|
||||
|
||||
private static final String[] LFS_VERSION_DOMAINS = {
|
||||
"hawser", "git-lfs"
|
||||
};
|
||||
|
||||
private static final String[] VALID_LFS_FILES = {
|
||||
// valid LFS pointer
|
||||
"version https://%s.github.com/spec/v1\n"
|
||||
+ "oid sha256:" + OID + "\n"
|
||||
+ "size " + SIZE + "\n",
|
||||
// valid LFS pointer with "custom" key
|
||||
"version https://%s.github.com/spec/v1\n"
|
||||
+ "custom key with value\n"
|
||||
+ "oid sha256:" + OID + "\n"
|
||||
+ "size " + SIZE + "\n",
|
||||
// valid LFS pointer with key with "."
|
||||
"version https://%s.github.com/spec/v1\n"
|
||||
+ "oid sha256:" + OID + "\n"
|
||||
+ "r.key key with .\n"
|
||||
+ "size " + SIZE + "\n",
|
||||
// valid LFS pointer with key with "-"
|
||||
"version https://%s.github.com/spec/v1\n"
|
||||
+ "oid sha256:" + OID + "\n"
|
||||
+ "size " + SIZE + "\n"
|
||||
+ "valid-name another valid key\n" };
|
||||
|
||||
@Test
|
||||
public void testRegularFilesInRepositoryRoot() throws Exception {
|
||||
for (String file : NOT_VALID_LFS_FILES) {
|
||||
assertLfs("file.bin", file).withRecursive(false).shouldBe(false);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNestedRegularFiles() throws Exception {
|
||||
for (String file : NOT_VALID_LFS_FILES) {
|
||||
assertLfs("a/file.bin", file).withRecursive(true).shouldBe(false);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testValidPointersInRepositoryRoot() throws Exception {
|
||||
for (String domain : LFS_VERSION_DOMAINS) {
|
||||
for (String file : VALID_LFS_FILES) {
|
||||
assertLfs("file.bin", String.format(file, domain))
|
||||
.withRecursive(true).shouldBe(true)
|
||||
.check();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testValidNestedPointers() throws Exception {
|
||||
for (String domain : LFS_VERSION_DOMAINS) {
|
||||
for (String file : VALID_LFS_FILES) {
|
||||
assertLfs("a/file.bin", String.format(file, domain))
|
||||
.withRecursive(true).shouldBe(true).check();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testValidNestedPointersWithoutRecurrence() throws Exception {
|
||||
for (String domain : LFS_VERSION_DOMAINS) {
|
||||
for (String file : VALID_LFS_FILES) {
|
||||
assertLfs("file.bin", String.format(file, domain))
|
||||
.withRecursive(false).shouldBe(true).check();
|
||||
assertLfs("a/file.bin", String.format(file, domain))
|
||||
.withRecursive(false).shouldBe(false).check();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static LfsTreeWalk assertLfs(String path, String content) {
|
||||
return new LfsTreeWalk(path, content);
|
||||
}
|
||||
|
||||
private static class LfsTreeWalk {
|
||||
private final String path;
|
||||
|
||||
private final String content;
|
||||
|
||||
private boolean state;
|
||||
|
||||
private boolean recursive;
|
||||
|
||||
private TestRepository<InMemoryRepository> tr;
|
||||
|
||||
LfsTreeWalk(String path, String content) {
|
||||
this.path = path;
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
LfsTreeWalk withRecursive(boolean shouldBeRecursive) {
|
||||
this.recursive = shouldBeRecursive;
|
||||
return this;
|
||||
}
|
||||
|
||||
LfsTreeWalk shouldBe(boolean shouldBeValid) {
|
||||
this.state = shouldBeValid;
|
||||
return this;
|
||||
}
|
||||
|
||||
void check() throws Exception {
|
||||
tr = new TestRepository<>(new InMemoryRepository(
|
||||
new DfsRepositoryDescription("test")));
|
||||
RevCommit commit = tr.branch("master").commit().add(path, content)
|
||||
.message("initial commit").create();
|
||||
RevTree tree = parseCommit(commit);
|
||||
LfsPointerFilter filter = new LfsPointerFilter();
|
||||
try (TreeWalk treeWalk = new TreeWalk(tr.getRepository())) {
|
||||
treeWalk.addTree(tree);
|
||||
treeWalk.setRecursive(recursive);
|
||||
treeWalk.setFilter(filter);
|
||||
|
||||
if (state) {
|
||||
assertTrue(treeWalk.next());
|
||||
assertEquals(path, treeWalk.getPathString());
|
||||
assertNotNull(filter.getPointer());
|
||||
assertEquals(SIZE, filter.getPointer().getSize());
|
||||
assertEquals(OID, filter.getPointer().getOid().name());
|
||||
} else {
|
||||
assertFalse(treeWalk.next());
|
||||
assertNull(filter.getPointer());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private RevTree parseCommit(RevCommit commit) throws Exception {
|
||||
try (ObjectWalk ow = new ObjectWalk(tr.getRepository())) {
|
||||
return ow.parseCommit(commit).getTree();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -12,7 +12,10 @@ Export-Package: org.eclipse.jgit.lfs;version="4.7.0",
|
|||
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
|
||||
Import-Package: org.eclipse.jgit.annotations;version="[4.7.0,4.8.0)";resolution:=optional,
|
||||
org.eclipse.jgit.attributes;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.errors;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.internal.storage.file;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.lib;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.nls;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.treewalk;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.treewalk.filter;version="[4.7.0,4.8.0)",
|
||||
org.eclipse.jgit.util;version="[4.7.0,4.8.0)"
|
||||
|
|
|
@ -68,6 +68,12 @@ public class LfsPointer {
|
|||
*/
|
||||
public static final String VERSION = "https://git-lfs.github.com/spec/v1"; //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* The version of the LfsPointer file format using legacy URL
|
||||
* @since 4.7
|
||||
*/
|
||||
public static final String VERSION_LEGACY = "https://hawser.github.com/spec/v1"; //$NON-NLS-1$
|
||||
|
||||
/**
|
||||
* The name of the hash function as used in the pointer files. This will
|
||||
* evaluate to "sha256"
|
||||
|
@ -150,14 +156,13 @@ public static LfsPointer parseLfsPointer(InputStream in)
|
|||
if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$
|
||||
continue;
|
||||
} else if (s.startsWith("version") && s.length() > 8 //$NON-NLS-1$
|
||||
&& s.substring(8).trim().equals(VERSION)) {
|
||||
&& (s.substring(8).trim().equals(VERSION) ||
|
||||
s.substring(8).trim().equals(VERSION_LEGACY))) {
|
||||
versionLine = true;
|
||||
} else if (s.startsWith("oid sha256:")) { //$NON-NLS-1$
|
||||
id = LongObjectId.fromString(s.substring(11).trim());
|
||||
} else if (s.startsWith("size") && s.length() > 5) { //$NON-NLS-1$
|
||||
sz = Long.parseLong(s.substring(5).trim());
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (versionLine && id != null && sz > -1) {
|
||||
|
|
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
* Copyright (C) 2015, 2017, Dariusz Luksza <dariusz@luksza.org>
|
||||
* and other copyright owners as documented in the project's IP log.
|
||||
*
|
||||
* This program and the accompanying materials are made available
|
||||
* under the terms of the Eclipse Distribution License v1.0 which
|
||||
* accompanies this distribution, is reproduced below, and is
|
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
*
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or
|
||||
* without modification, are permitted provided that the following
|
||||
* conditions are met:
|
||||
*
|
||||
* - Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
*
|
||||
* - Redistributions in binary form must reproduce the above
|
||||
* copyright notice, this list of conditions and the following
|
||||
* disclaimer in the documentation and/or other materials provided
|
||||
* with the distribution.
|
||||
*
|
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the
|
||||
* names of its contributors may be used to endorse or promote
|
||||
* products derived from this software without specific prior
|
||||
* written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
|
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
package org.eclipse.jgit.lfs.lib;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
|
||||
import org.eclipse.jgit.errors.MissingObjectException;
|
||||
import org.eclipse.jgit.lfs.LfsPointer;
|
||||
import org.eclipse.jgit.lib.ObjectId;
|
||||
import org.eclipse.jgit.lib.ObjectLoader;
|
||||
import org.eclipse.jgit.lib.ObjectStream;
|
||||
import org.eclipse.jgit.treewalk.TreeWalk;
|
||||
import org.eclipse.jgit.treewalk.filter.TreeFilter;
|
||||
|
||||
/**
|
||||
* Detects Large File pointers, as described in [1] in Git repository.
|
||||
*
|
||||
* [1] https://github.com/github/git-lfs/blob/master/docs/spec.md
|
||||
*
|
||||
* @since 4.7
|
||||
*/
|
||||
public class LfsPointerFilter extends TreeFilter {
|
||||
|
||||
private LfsPointer pointer;
|
||||
|
||||
/**
|
||||
* @return {@link LfsPointer} or {@code null}
|
||||
*/
|
||||
public LfsPointer getPointer() {
|
||||
return pointer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean include(TreeWalk walk) throws MissingObjectException,
|
||||
IncorrectObjectTypeException, IOException {
|
||||
pointer = null;
|
||||
if (walk.isSubtree()) {
|
||||
return walk.isRecursive();
|
||||
}
|
||||
ObjectId objectId = walk.getObjectId(0);
|
||||
ObjectLoader object = walk.getObjectReader().open(objectId);
|
||||
if (object.getSize() > 1024) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try (ObjectStream stream = object.openStream()) {
|
||||
pointer = LfsPointer.parseLfsPointer(stream);
|
||||
return pointer != null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean shouldBeRecursive() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TreeFilter clone() {
|
||||
return new LfsPointerFilter();
|
||||
}
|
||||
}
|
|
@ -60,7 +60,11 @@ Export-Package: org.eclipse.jgit.annotations;version="4.7.0",
|
|||
org.eclipse.jgit.ignore.internal;version="4.7.0";x-friends:="org.eclipse.jgit.test",
|
||||
org.eclipse.jgit.internal;version="4.7.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test",
|
||||
org.eclipse.jgit.internal.ketch;version="4.7.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
|
||||
org.eclipse.jgit.internal.storage.dfs;version="4.7.0";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.server,org.eclipse.jgit.http.test",
|
||||
org.eclipse.jgit.internal.storage.dfs;version="4.7.0";
|
||||
x-friends:="org.eclipse.jgit.test,
|
||||
org.eclipse.jgit.http.server,
|
||||
org.eclipse.jgit.http.test,
|
||||
org.eclipse.jgit.lfs.test",
|
||||
org.eclipse.jgit.internal.storage.file;version="4.7.0";
|
||||
x-friends:="org.eclipse.jgit.test,
|
||||
org.eclipse.jgit.junit,
|
||||
|
|
Loading…
Reference in New Issue