diff --git a/DEPENDENCIES b/DEPENDENCIES new file mode 100644 index 000000000..bffe3d928 --- /dev/null +++ b/DEPENDENCIES @@ -0,0 +1,66 @@ +maven/mavencentral/args4j/args4j/2.33, MIT, approved, CQ11068 +maven/mavencentral/com.google.code.gson/gson/2.8.6, Apache-2.0, approved, CQ23102 +maven/mavencentral/com.googlecode.javaewah/JavaEWAH/1.1.7, Apache-2.0, approved, CQ11658 +maven/mavencentral/com.jcraft/jsch/0.1.55, BSD-3-Clause, approved, CQ19435 +maven/mavencentral/com.jcraft/jzlib/1.1.1, BSD-2-Clause, approved, CQ6218 +maven/mavencentral/commons-codec/commons-codec/1.11, Apache-2.0, approved, CQ15971 +maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162 +maven/mavencentral/javax.servlet/javax.servlet-api/3.1.0, Apache-2.0 AND (CDDL-1.1 OR GPL-2.0 WITH Classpath-exception-2.0), approved, emo_ip_team +maven/mavencentral/junit/junit/4.13, , approved, CQ22796 +maven/mavencentral/log4j/log4j/1.2.15, Apache-2.0, approved, CQ7837 +maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.9.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/net.bytebuddy/byte-buddy/1.9.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/net.i2p.crypto/eddsa/0.3.0, CC0, approved, CQ17804 +maven/mavencentral/net.sf.jopt-simple/jopt-simple/4.6, MIT, approved, clearlydefined +maven/mavencentral/org.apache.ant/ant-launcher/1.10.8, Apache-2.0 AND W3C AND LicenseRef-Public-Domain, approved, CQ15560 +maven/mavencentral/org.apache.ant/ant/1.10.8, Apache-2.0 AND W3C AND LicenseRef-Public-Domain, approved, CQ15560 +maven/mavencentral/org.apache.commons/commons-compress/1.19, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.commons/commons-math3/3.2, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.httpcomponents/httpclient/4.5.13, Apache-2.0, approved, CQ22761 +maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approved, CQ18704 +maven/mavencentral/org.apache.sshd/sshd-common/2.6.0, Apache-2.0 AND ISC, approved, CQ22992 +maven/mavencentral/org.apache.sshd/sshd-core/2.6.0, Apache-2.0 AND ISC, approved, CQ22992 +maven/mavencentral/org.apache.sshd/sshd-osgi/2.6.0, Apache-2.0 AND ISC, approved, CQ22992 +maven/mavencentral/org.apache.sshd/sshd-sftp/2.6.0, Apache-2.0 AND ISC, approved, CQ22993 +maven/mavencentral/org.assertj/assertj-core/3.14.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.bouncycastle/bcpg-jdk15on/1.65, Apache-2.0, approved, CQ21975 +maven/mavencentral/org.bouncycastle/bcpkix-jdk15on/1.65, MIT AND LicenseRef-Public-Domain, approved, CQ21976 +maven/mavencentral/org.bouncycastle/bcprov-jdk15on/1.65.01, MIT AND LicenseRef-Public-Domain, approved, CQ21977 +maven/mavencentral/org.eclipse.jetty/jetty-http/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-io/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-security/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-server/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-servlet/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-util-ajax/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-util/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ant.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ant/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.archive/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.gpg.bc/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.http.apache/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.http.server/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.http.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.junit.http/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.junit.ssh/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.junit/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs.server.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs.server/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.pgm.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.pgm/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ssh.apache.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ssh.apache/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ssh.jsch/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ui/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.hamcrest/hamcrest-core/1.3, BSD-2-Clause, approved, CQ7063 +maven/mavencentral/org.mockito/mockito-core/2.23.0, MIT, approved, CQ17976 +maven/mavencentral/org.objenesis/objenesis/2.6, Apache-2.0, approved, CQ15478 +maven/mavencentral/org.openjdk.jmh/jmh-core/1.21, GPL-2.0, approved, CQ20517 +maven/mavencentral/org.openjdk.jmh/jmh-generator-annprocess/1.21, GPL-2.0, approved, CQ20518 +maven/mavencentral/org.osgi/org.osgi.core/4.3.1, Apache-2.0, approved, CQ10111 +maven/mavencentral/org.slf4j/slf4j-api/1.7.30, MIT, approved, CQ13368 +maven/mavencentral/org.slf4j/slf4j-log4j12/1.7.30, MIT, approved, CQ7665 +maven/mavencentral/org.tukaani/xz/1.8, LicenseRef-Public-Domain, approved, CQ15386 diff --git a/Documentation/config-options.md b/Documentation/config-options.md index d46355176..a9ca48c6a 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -7,6 +7,8 @@ | ✅ | option defined by native git | | ⃞ | jgit custom option not supported by native git | +For details on native git options see also the official [git config documentation](https://git-scm.com/docs/git-config). + ## __core__ options | option | default | git option | description | @@ -59,6 +61,24 @@ | `gc.pruneExpire` | `2.weeks.ago` | ✅ | Grace period after which unreachable objects will be pruned. | | `gc.prunePackExpire` | `1.hour.ago` | ⃞ | Grace period after which packfiles only containing unreachable objects will be pruned. | +## __http__ options + +| option | default | git option | description | +|---------|---------|------------|-------------| +| `http.cookieFile`| | ✅ | Absolute path (with tilde expansion) of a cookie file in Netscape format. | +| `http.cookieFileCacheLimit`| 10 | ⃞ | JGit caches at most this number of the most recently used cookie files. | +| `http.extraHeader`| | ✅ | Extra HTTP header(s) to send with HTTP requests, in the format "`Key: Value`". May appear multiple times; an empty option clears the list. | +| `http.followRedirects`| `initial` | ✅ | `true`, `false`, or `initial`. Whether to follow a redirect always, never, or only on the first HTTP request in a git remote operation. | +| `http.maxRedirects`| 5 | ⃞ | Maximum number of redirects to follow; can be overridden via the Java system property `http.maxRedirects`. | +| `http.postBuffer`| `1 MiB` | ✅ | Maximum size in bytes for single HTTP POST requests; for larger requests, HTTP 1.1 chunked transfer is used. | +| `http.saveCookies`| `false` | ✅ | Boolean; if `true` and `http.cookieFile` is set, save received cookies. | +| `http.sslVerify`| `true` | ✅ | Boolean; whether to check SSL certificates in HTTPS connections. | +| `http.userAgent`| | ✅ | User-agent string to send with HTTP requests. Must be 7bit-ASCII. Can be overridden via environment variable `GIT_HTTP_USER_AGENT`. | + +All `http.*` options can also be specified in a URL-specific way using the format `http..*`. See the official [git config documentation](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httplturlgt) for details. + +Proxy configuration uses the standard Java mechanisms via class `java.net.ProxySelector`. + ## __pack__ options | option | default | git option | description | diff --git a/org.eclipse.jgit.archive/.settings/.api_filters b/org.eclipse.jgit.archive/.settings/.api_filters deleted file mode 100644 index f4a934aeb..000000000 --- a/org.eclipse.jgit.archive/.settings/.api_filters +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index 64556acc1..5622108dc 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -25,6 +25,7 @@ import java.io.Reader; import java.nio.file.Path; import java.time.Instant; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -39,6 +40,7 @@ import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; @@ -385,6 +387,16 @@ protected void createBranch(ObjectId objectId, String branchName) updateRef.update(); } + /** + * Get all Refs + * + * @return list of refs + * @throws IOException + */ + public List getRefs() throws IOException { + return db.getRefDatabase().getRefs(); + } + /** * Checkout a branch * diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index e3eb2c536..0232156a4 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java @@ -44,7 +44,9 @@ import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.internal.storage.file.ObjectDirectory; import org.eclipse.jgit.internal.storage.file.Pack; +import org.eclipse.jgit.internal.storage.file.PackFile; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -906,23 +908,22 @@ public void packAndPrune() throws Exception { ObjectDirectory odb = (ObjectDirectory) db.getObjectDatabase(); NullProgressMonitor m = NullProgressMonitor.INSTANCE; - final File pack, idx; + final PackFile pack, idx; try (PackWriter pw = new PackWriter(db)) { Set all = new HashSet<>(); for (Ref r : db.getRefDatabase().getRefs()) all.add(r.getObjectId()); pw.preparePack(m, all, PackWriter.NONE); - final ObjectId name = pw.computeName(); - - pack = nameFor(odb, name, ".pack"); + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(pack))) { pw.writePack(m, m, out); } pack.setReadOnly(); - idx = nameFor(odb, name, ".idx"); + idx = pack.create(PackExt.INDEX); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(idx))) { pw.writeIndex(out); @@ -960,11 +961,6 @@ private static void prunePacked(ObjectDirectory odb) throws IOException { } } - private static File nameFor(ObjectDirectory odb, ObjectId name, String t) { - File packdir = odb.getPackDirectory(); - return new File(packdir, "pack-" + name.name() + t); - } - private void writeFile(File p, byte[] bin) throws IOException, ObjectWritingException { final LockFile lck = new LockFile(p); diff --git a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF index 42bebbf82..24abf78ef 100644 --- a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF @@ -7,7 +7,9 @@ Bundle-Version: 6.0.0.qualifier Bundle-Vendor: %Bundle-Vendor Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.eclipse.jgit.internal.storage.dfs;version="[6.0.0,6.1.0)", +Import-Package: org.eclipse.jgit.api;version="[6.0.0,6.1.0)", + org.eclipse.jgit.attributes;version="[6.0.0,6.1.0)", + org.eclipse.jgit.internal.storage.dfs;version="[6.0.0,6.1.0)", org.eclipse.jgit.junit;version="[6.0.0,6.1.0)", org.eclipse.jgit.lfs;version="[6.0.0,6.1.0)", org.eclipse.jgit.lfs.errors;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java new file mode 100644 index 000000000..8964310e4 --- /dev/null +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ResetCommand.ResetType; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.StoredConfig; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LfsGitTest extends RepositoryTestCase { + + private static final String SMUDGE_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE; + + private static final String CLEAN_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN; + + @BeforeClass + public static void installLfs() { + FilterCommandRegistry.register(SMUDGE_NAME, SmudgeFilter.FACTORY); + FilterCommandRegistry.register(CLEAN_NAME, CleanFilter.FACTORY); + } + + @AfterClass + public static void removeLfs() { + FilterCommandRegistry.unregister(SMUDGE_NAME); + FilterCommandRegistry.unregister(CLEAN_NAME); + } + + private Git git; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + git = new Git(db); + // commit something + writeTrashFile("Test.txt", "Hello world"); + git.add().addFilepattern("Test.txt").call(); + git.commit().setMessage("Initial commit").call(); + // prepare the config for LFS + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "lfs", "clean", CLEAN_NAME); + config.setString("filter", "lfs", "smudge", SMUDGE_NAME); + config.save(); + } + + @Test + public void checkoutNonLfsPointer() throws Exception { + String content = "size_t\nsome_function(void* ptr);\n"; + File smallFile = writeTrashFile("Test.txt", content); + StringBuilder largeContent = new StringBuilder( + LfsPointer.SIZE_THRESHOLD * 4); + while (largeContent.length() < LfsPointer.SIZE_THRESHOLD * 4) { + largeContent.append(content); + } + File largeFile = writeTrashFile("large.txt", largeContent.toString()); + fsTick(largeFile); + git.add().addFilepattern("Test.txt").addFilepattern("large.txt").call(); + git.commit().setMessage("Text files").call(); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("attributes").call(); + assertTrue(smallFile.delete()); + assertTrue(largeFile.delete()); + // This reset will run the two text files through the smudge filter + git.reset().setMode(ResetType.HARD).call(); + assertTrue(smallFile.exists()); + assertTrue(largeFile.exists()); + checkFile(smallFile, content); + checkFile(largeFile, largeContent.toString()); + // Modify the large file + largeContent.append(content); + writeTrashFile("large.txt", largeContent.toString()); + // This should convert largeFile to an LFS pointer + git.add().addFilepattern("large.txt").call(); + git.commit().setMessage("Large modified").call(); + String lfsPtr = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:d041ab19bd7edd899b3c0450d0f61819f96672f0b22d26c9753abc62e1261614\n" + + "size 858\n"; + assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]" + + "[Test.txt, mode:100644, content:" + content + ']' + + "[large.txt, mode:100644, content:" + lfsPtr + ']', + indexState(CONTENT)); + // Verify the file has been saved + File savedFile = new File(db.getDirectory(), "lfs"); + savedFile = new File(savedFile, "objects"); + savedFile = new File(savedFile, "d0"); + savedFile = new File(savedFile, "41"); + savedFile = new File(savedFile, + "d041ab19bd7edd899b3c0450d0f61819f96672f0b22d26c9753abc62e1261614"); + String saved = new String(Files.readAllBytes(savedFile.toPath()), + StandardCharsets.UTF_8); + assertEquals(saved, largeContent.toString()); + + assertTrue(smallFile.delete()); + assertTrue(largeFile.delete()); + git.reset().setMode(ResetType.HARD).call(); + assertTrue(smallFile.exists()); + assertTrue(largeFile.exists()); + checkFile(smallFile, content); + checkFile(largeFile, largeContent.toString()); + assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]" + + "[Test.txt, mode:100644, content:" + content + ']' + + "[large.txt, mode:100644, content:" + lfsPtr + ']', + indexState(CONTENT)); + git.add().addFilepattern("Test.txt").call(); + git.commit().setMessage("Small committed again").call(); + String lfsPtrSmall = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:9110463275fb0e2f0e9fdeaf84e598e62915666161145cf08927079119cc7814\n" + + "size 33\n"; + assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]" + + "[Test.txt, mode:100644, content:" + lfsPtrSmall + ']' + + "[large.txt, mode:100644, content:" + lfsPtr + ']', + indexState(CONTENT)); + + assertTrue(git.status().call().isClean()); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java index 52c3001b8..032a19b5d 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017, Markus Duft and others + * Copyright (C) 2017, 2021 Markus Duft and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -45,7 +45,7 @@ public class LfsBlobFilter { */ public static ObjectLoader smudgeLfsBlob(Repository db, ObjectLoader loader) throws IOException { - if (loader.getSize() > LfsPointer.SIZE_THRESHOLD) { + if (loader.getSize() > LfsPointer.FULL_SIZE_THRESHOLD) { return loader; } diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java index aef441638..0a8a3faec 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -11,7 +11,9 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -25,6 +27,7 @@ import org.eclipse.jgit.lfs.lib.AnyLongObjectId; import org.eclipse.jgit.lfs.lib.Constants; import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.util.IO; /** * Represents an LFS pointer file @@ -57,6 +60,12 @@ public class LfsPointer implements Comparable { public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION .toLowerCase(Locale.ROOT).replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$ + /** + * {@link #SIZE_THRESHOLD} is too low; with lfs extensions a LFS pointer can + * be larger. But 8kB should be more than enough. + */ + static final int FULL_SIZE_THRESHOLD = 8 * 1024; + private final AnyLongObjectId oid; private final long size; @@ -115,64 +124,113 @@ public void encode(OutputStream out) { /** * Try to parse the data provided by an InputStream to the format defined by - * {@link #VERSION} + * {@link #VERSION}. If the given stream supports mark and reset as + * indicated by {@link InputStream#markSupported()}, its input position will + * be reset if the stream content is not actually a LFS pointer (i.e., when + * {@code null} is returned). If the stream content is an invalid LFS + * pointer or the given stream does not support mark/reset, the input + * position may not be reset. * * @param in * the {@link java.io.InputStream} from where to read the data - * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or null - * if the stream was not parseable as LfsPointer + * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or {@code null} if the + * stream was not parseable as LfsPointer * @throws java.io.IOException */ @Nullable public static LfsPointer parseLfsPointer(InputStream in) throws IOException { + if (in.markSupported()) { + return parse(in); + } + // Fallback; note that while parse() resets its input stream, that won't + // reset "in". + return parse(new BufferedInputStream(in)); + } + + @Nullable + private static LfsPointer parse(InputStream in) + throws IOException { + if (!in.markSupported()) { + // No translation; internal error + throw new IllegalArgumentException( + "LFS pointer parsing needs InputStream.markSupported() == true"); //$NON-NLS-1$ + } + // Try reading only a short block first. + in.mark(SIZE_THRESHOLD); + byte[] preamble = new byte[SIZE_THRESHOLD]; + int length = IO.readFully(in, preamble, 0); + if (length < preamble.length || in.read() < 0) { + // We have the whole file. Try to parse a pointer from it. + try (BufferedReader r = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(preamble, 0, length), UTF_8))) { + LfsPointer ptr = parse(r); + if (ptr == null) { + in.reset(); + } + return ptr; + } + } + // Longer than SIZE_THRESHOLD: expect "version" to be the first line. + boolean hasVersion = checkVersion(preamble); + in.reset(); + if (!hasVersion) { + return null; + } + in.mark(FULL_SIZE_THRESHOLD); + byte[] fullPointer = new byte[FULL_SIZE_THRESHOLD]; + length = IO.readFully(in, fullPointer, 0); + if (length == fullPointer.length && in.read() >= 0) { + in.reset(); + return null; // Too long. + } + try (BufferedReader r = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(fullPointer, 0, length), UTF_8))) { + LfsPointer ptr = parse(r); + if (ptr == null) { + in.reset(); + } + return ptr; + } + } + + private static LfsPointer parse(BufferedReader r) throws IOException { boolean versionLine = false; LongObjectId id = null; long sz = -1; - // This parsing is a bit too general if we go by the spec at // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - // Comment lines are not mentioned in the spec, and the "version" line - // MUST be the first. - try (BufferedReader br = new BufferedReader( - new InputStreamReader(in, UTF_8))) { - for (String s = br.readLine(); s != null; s = br.readLine()) { - if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ - continue; - } else if (s.startsWith("version")) { //$NON-NLS-1$ - if (versionLine || s.length() < 8 || s.charAt(7) != ' ') { - return null; // Not a LFS pointer - } - String rest = s.substring(8).trim(); - versionLine = VERSION.equals(rest) - || VERSION_LEGACY.equals(rest); - if (!versionLine) { - return null; // Not a LFS pointer - } - } else { - try { - if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ - if (id != null) { - return null; // Not a LFS pointer - } - id = LongObjectId - .fromString(s.substring(11).trim()); - } else if (s.startsWith("size")) { //$NON-NLS-1$ - if (sz > 0 || s.length() < 5 - || s.charAt(4) != ' ') { - return null; // Not a LFS pointer - } - sz = Long.parseLong(s.substring(5).trim()); + // Comment lines are not mentioned in the spec, the "version" line + // MUST be the first, and keys are ordered alphabetically. + for (String s = r.readLine(); s != null; s = r.readLine()) { + if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ + continue; + } else if (s.startsWith("version")) { //$NON-NLS-1$ + if (versionLine || !checkVersionLine(s)) { + return null; // Not a LFS pointer + } + versionLine = true; + } else { + try { + if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ + if (id != null) { + return null; // Not a LFS pointer } - } catch (RuntimeException e) { - // We could not parse the line. If we have a version - // already, this is a corrupt LFS pointer. Otherwise it - // is just not an LFS pointer. - if (versionLine) { - throw e; + id = LongObjectId.fromString(s.substring(11).trim()); + } else if (s.startsWith("size")) { //$NON-NLS-1$ + if (sz > 0 || s.length() < 5 || s.charAt(4) != ' ') { + return null; // Not a LFS pointer } - return null; + sz = Long.parseLong(s.substring(5).trim()); } + } catch (RuntimeException e) { + // We could not parse the line. If we have a version + // already, this is a corrupt LFS pointer. Otherwise it + // is just not an LFS pointer. + if (versionLine) { + throw e; + } + return null; } } if (versionLine && id != null && sz > -1) { @@ -182,6 +240,30 @@ public static LfsPointer parseLfsPointer(InputStream in) return null; } + private static boolean checkVersion(byte[] data) { + // According to the spec at + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + // it MUST always be the first line. + try (BufferedReader r = new BufferedReader( + new InputStreamReader(new ByteArrayInputStream(data), UTF_8))) { + String s = r.readLine(); + if (s != null && s.startsWith("version")) { //$NON-NLS-1$ + return checkVersionLine(s); + } + } catch (IOException e) { + // Doesn't occur, we're reading from a byte array! + } + return false; + } + + private static boolean checkVersionLine(String s) { + if (s.length() < 8 || s.charAt(7) != ' ') { + return false; // Not a valid LFS pointer version line + } + String rest = s.substring(8).trim(); + return VERSION.equals(rest) || VERSION_LEGACY.equals(rest); + } + /** {@inheritDoc} */ @Override public String toString() { diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java index 2f80d5b9a..341188756 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016, Christian Halstrick and others + * Copyright (C) 2016, 2021 Christian Halstrick and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -11,6 +11,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -87,20 +88,31 @@ static void register() { */ public SmudgeFilter(Repository db, InputStream in, OutputStream out) throws IOException { + this(in.markSupported() ? in : new BufferedInputStream(in), out, db); + } + + private SmudgeFilter(InputStream in, OutputStream out, Repository db) + throws IOException { super(in, out); + InputStream from = in; try { - Lfs lfs = new Lfs(db); - LfsPointer res = LfsPointer.parseLfsPointer(in); + LfsPointer res = LfsPointer.parseLfsPointer(from); if (res != null) { AnyLongObjectId oid = res.getOid(); + Lfs lfs = new Lfs(db); Path mediaFile = lfs.getMediaFile(oid); if (!Files.exists(mediaFile)) { downloadLfsResource(lfs, db, res); } this.in = Files.newInputStream(mediaFile); + } else { + // Not swapped; stream was reset, don't close! + from = null; } } finally { - in.close(); // make sure the swapped stream is closed properly. + if (from != null) { + from.close(); // Close the swapped-out stream + } } } diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java index d84eebd22..99bae49ab 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015, 2017, Dariusz Luksza and others + * Copyright (C) 2015, 2021 Dariusz Luksza and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -58,6 +58,8 @@ public boolean include(TreeWalk walk) throws MissingObjectException, try (ObjectStream stream = object.openStream()) { pointer = LfsPointer.parseLfsPointer(stream); return pointer != null; + } catch (RuntimeException e) { + return false; } } diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target index 68378a2b0..715986b4b 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd index fb1ac6b25..ed443a6fb 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd @@ -1,7 +1,7 @@ target "jgit-4.10" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2018-12/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target index 18d525d8e..192671249 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd index 0d5628063..013d6218b 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd @@ -1,7 +1,7 @@ target "jgit-4.11" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2019-03/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target index d72f08d29..4449dc3f6 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd index 5a024152d..99008ab8c 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd @@ -1,7 +1,7 @@ target "jgit-4.12" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2019-06/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target index d0e559271..01a10e712 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd index 84e5c25ef..d0db92c59 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd @@ -1,7 +1,7 @@ target "jgit-4.13" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2019-09/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target index 42278f6ef..b56f9a1f8 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd index 6d793a607..e0a730e67 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd @@ -1,7 +1,7 @@ target "jgit-4.14" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2019-12/201912181000/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target index 0d5166e16..f3820a780 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd index 4ce832bf9..773a9a9ba 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd @@ -1,7 +1,7 @@ target "jgit-4.15" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2020-03/202003181000/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target index b4d53069b..6a9f58291 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd index 1b56447ce..8b4de8bb3 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd @@ -1,7 +1,7 @@ target "jgit-4.16" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2020-06/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target index 47fc74be6..b7481e09f 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd index 367020ce0..b2585be73 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd @@ -1,7 +1,7 @@ target "jgit-4.17" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2020-09/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target index b393e6075..6d851a252 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd index 507ddd1dc..6d16256dc 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd @@ -1,7 +1,7 @@ target "jgit-4.18" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2020-12/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target index f37692616..1a0505de9 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd index 3b1b19c84..7ed5377d5 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd @@ -1,7 +1,7 @@ target "jgit-4.19-staging" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/staging/2021-03/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target index 26715ee18..249be4cf3 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd index 23bf87c07..6e7cd8b66 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd @@ -1,7 +1,7 @@ target "jgit-4.6" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/neon/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target index 64fe05495..72c44d7b4 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd index c33e4a39b..5a58b006e 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd @@ -1,7 +1,7 @@ target "jgit-4.7" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/oxygen/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target index f7a3a3b26..10e3deaf7 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd index c40bacdb8..31148776f 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd @@ -1,7 +1,7 @@ target "jgit-4.8" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/photon/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target index 4afbe9973..55b3c6056 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target @@ -1,7 +1,7 @@ - + @@ -86,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd index 5aa63be64..132a0b06f 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd @@ -1,7 +1,7 @@ target "jgit-4.9" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20210216215844.tpd" +include "orbit/R20210223232630-2021-03.tpd" location "https://download.eclipse.org/releases/2018-09/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20210216215844.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20210223232630-2021-03.tpd similarity index 97% rename from org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20210216215844.tpd rename to org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20210223232630-2021-03.tpd index 29e5bc800..605a43bb1 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20210216215844.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20210223232630-2021-03.tpd @@ -1,7 +1,7 @@ -target "S20210216215844" with source configurePhase +target "R20210223232630-2021-03" with source configurePhase // see https://download.eclipse.org/tools/orbit/downloads/ -location "https://download.eclipse.org/tools/orbit/downloads/drops/S20210216215844/repository" { +location "https://download.eclipse.org/tools/orbit/downloads/drops/R20210223232630/repository" { com.google.gson [2.8.6.v20201231-1626,2.8.6.v20201231-1626] com.google.gson.source [2.8.6.v20201231-1626,2.8.6.v20201231-1626] com.jcraft.jsch [0.1.55.v20190404-1902,0.1.55.v20190404-1902] diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple1.txt b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple1.txt index e06b38c71..527893acf 100644 --- a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple1.txt +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple1.txt @@ -1,2 +1,2 @@ -some-domain1 TRUE /some/path1 FALSE 1893499200000 key1 valueFromSimple1 -some-domain1 TRUE /some/path1 FALSE 1893499200000 key2 valueFromSimple1 \ No newline at end of file +some-domain1 TRUE /some/path1 FALSE 1893499200 key1 valueFromSimple1 +some-domain1 TRUE /some/path1 FALSE 1893499200 key2 valueFromSimple1 \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple2.txt b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple2.txt index 4bf6723fd..5ec060627 100644 --- a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple2.txt +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-simple2.txt @@ -1,2 +1,2 @@ -some-domain1 TRUE /some/path1 FALSE 1893499200000 key1 valueFromSimple2 -some-domain1 TRUE /some/path1 FALSE 1893499200000 key3 valueFromSimple2 \ No newline at end of file +some-domain1 TRUE /some/path1 FALSE 1893499200 key1 valueFromSimple2 +some-domain1 TRUE /some/path1 FALSE 1893499200 key3 valueFromSimple2 \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-empty-and-comment-lines.txt b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-empty-and-comment-lines.txt index a9b8a2815..573ee9ee1 100644 --- a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-empty-and-comment-lines.txt +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-empty-and-comment-lines.txt @@ -3,6 +3,6 @@ some-domain1 TRUE /some/path1 FALSE 0 key1 value1 # expires date is 01/01/2030 @ 12:00am (UTC) -#HttpOnly_.some-domain2 TRUE /some/path2 TRUE 1893499200000 key2 value2 +#HttpOnly_.some-domain2 TRUE /some/path2 TRUE 1893499200 key2 value2 -some-domain3 TRUE /some/path3 FALSE 1893499200000 key3 value3 \ No newline at end of file +some-domain3 TRUE /some/path3 FALSE 1893499200 key3 value3 \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-milliseconds.txt b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-milliseconds.txt new file mode 100644 index 000000000..940e3b1a5 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/internal/transport/http/cookies-with-milliseconds.txt @@ -0,0 +1,2 @@ +some-domain1 TRUE /some/path1 FALSE 1893499200000 key1 valueFromSimple1 +some-domain1 TRUE /some/path1 FALSE 1893499200 key2 valueFromSimple1 \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RenameDetectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RenameDetectorTest.java index 6203feda4..2ea3cd7eb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RenameDetectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RenameDetectorTest.java @@ -542,6 +542,43 @@ public void testBreakModify_RejoinIfUnpaired() throws Exception { assertEquals(0, modify.score); } + @Test + public void testExactRename_LargeFile() throws Exception { + ObjectId aId = blob("blah\nblah\nfoo"); // size = 14 + + DiffEntry a = DiffEntry.add(PATH_A, aId); + DiffEntry b = DiffEntry.delete(PATH_Q, aId); + + rd.add(a); + rd.add(b); + + // Exact renames are identified for large files + rd.setBigFileThreshold(10); + List entries = rd.compute(); + assertEquals(1, entries.size()); + assertRename(b, a, 100, entries.get(0)); + } + + @Test + public void testInexactRename_LargeFile() throws Exception { + ObjectId aId = blob("blah\nblah\nfoo"); // size = 14 + ObjectId bId = blob("bla\nblah\nfoo"); // size = 13 + + DiffEntry a = DiffEntry.add(PATH_A, aId); + DiffEntry b = DiffEntry.delete(PATH_Q, bId); + + rd.add(a); + rd.add(b); + + rd.setBigFileThreshold(10); + + // Inexact renames are not detected for large files + List entries = rd.compute(); + assertEquals(2, entries.size()); + assertAdd(PATH_A, aId, FileMode.REGULAR_FILE, entries.get(0)); + assertDelete(PATH_Q, bId, FileMode.REGULAR_FILE, entries.get(1)); + } + @Test public void testSetRenameScore_IllegalArgs() throws Exception { try { @@ -634,4 +671,15 @@ private static void assertAdd(String newName, ObjectId newId, assertEquals(AbbreviatedObjectId.fromObjectId(newId), add.newId); assertEquals(newMode, add.newMode); } + + private static void assertDelete(String oldName, ObjectId oldId, + FileMode oldMode, DiffEntry delete) { + assertEquals(DiffEntry.DEV_NULL, delete.newPath); + assertEquals(DiffEntry.A_ZERO, delete.newId); + assertEquals(FileMode.MISSING, delete.newMode); + assertEquals(ChangeType.DELETE, delete.changeType); + assertEquals(oldName, delete.oldPath); + assertEquals(AbbreviatedObjectId.fromObjectId(oldId), delete.oldId); + assertEquals(oldMode, delete.oldMode); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java index 45d864d45..bd36337f3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java @@ -28,6 +28,7 @@ import java.util.List; import org.eclipse.jgit.errors.AmbiguousObjectException; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.AbbreviatedObjectId; @@ -144,10 +145,9 @@ public void testAbbreviateIsActuallyUnique() throws Exception { objects.add(new PackedObjectInfo(ObjectId.fromRaw(idBuf))); } - String packName = "pack-" + id.name(); File packDir = db.getObjectDatabase().getPackDirectory(); - File idxFile = new File(packDir, packName + ".idx"); - File packFile = new File(packDir, packName + ".pack"); + PackFile idxFile = new PackFile(packDir, id, PackExt.INDEX); + PackFile packFile = idxFile.create(PackExt.PACK); FileUtils.mkdir(packDir, true); try (OutputStream dst = new BufferedOutputStream( new FileOutputStream(idxFile))) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java index da3b5bbc3..df5d952ee 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java @@ -27,6 +27,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.AnyObjectId; @@ -193,9 +194,10 @@ private File[] pack(Repository src, RevObject... list) pw.addObject(o); } - final ObjectId name = pw.computeName(); - final File packFile = fullPackFileName(name, ".pack"); - final File idxFile = fullPackFileName(name, ".idx"); + PackFile packFile = new PackFile( + db.getObjectDatabase().getPackDirectory(), pw.computeName(), + PackExt.PACK); + PackFile idxFile = packFile.create(PackExt.INDEX); final File[] files = new File[] { packFile, idxFile }; write(files, pw); return files; @@ -242,11 +244,6 @@ private static void touch(Instant begin, File dir) throws IOException { } } - private File fullPackFileName(ObjectId name, String suffix) { - final File packdir = db.getObjectDatabase().getPackDirectory(); - return new File(packdir, "pack-" + name.name() + suffix); - } - private RevObject writeBlob(Repository repo, String data) throws IOException { final byte[] bytes = Constants.encode(data); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index 42e423845..8dc1ddb9f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -23,6 +23,7 @@ import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -295,7 +296,7 @@ private void testPreserveOldPacks() throws Exception { // pack loose object into packfile gc.setExpireAgeMillis(0); gc.gc(); - File oldPackfile = tr.getRepository().getObjectDatabase().getPacks() + PackFile oldPackfile = tr.getRepository().getObjectDatabase().getPacks() .iterator().next().getPackFile(); assertTrue(oldPackfile.exists()); @@ -309,12 +310,59 @@ private void testPreserveOldPacks() throws Exception { configureGc(gc, false).setPreserveOldPacks(true); gc.gc(); - File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); - String oldPackFileName = oldPackfile.getName(); - String oldPackName = oldPackFileName.substring(0, - oldPackFileName.lastIndexOf('.')) + ".old-pack"; //$NON-NLS-1$ - File preservePackFile = new File(oldPackDir, oldPackName); - assertTrue(preservePackFile.exists()); + File preservedPackFile = oldPackfile.createPreservedForDirectory( + repo.getObjectDatabase().getPreservedDirectory()); + assertTrue(preservedPackFile.exists()); + } + + @Test + public void testPruneAndRestoreOldPacks() throws Exception { + String tempRef = "refs/heads/soon-to-be-unreferenced"; + BranchBuilder bb = tr.branch(tempRef); + bb.commit().add("A", "A").add("B", "B").create(); + + // Verify setup conditions + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + + // Force all referenced objects into packs (to avoid having loose objects) + configureGc(gc, false); + gc.setExpireAgeMillis(0); + gc.setPackExpireAgeMillis(0); + gc.gc(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + + // Delete the temp ref, orphaning its commit + RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setForceUpdate(true); + ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it! + RefUpdate.Result result = update.delete(); + assertEquals(RefUpdate.Result.FORCED, result); + + fsTick(); + + // Repack with only orphaned commit, so packfile will be pruned + configureGc(gc, false).setPreserveOldPacks(true); + gc.gc(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + assertEquals(0, stats.numberOfPackFiles); + + // Restore the temp ref to the deleted commit, should restore old-packs! + update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setNewObjectId(objectId); + update.setExpectedOldObjectId(null); + result = update.update(); + assertEquals(RefUpdate.Result.NEW, result); + + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); } private PackConfig configureGc(GC myGc, boolean aggressive) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java index 8472983d5..5fcdd3757 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java @@ -14,10 +14,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.File; import java.util.Iterator; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.junit.Test; @@ -40,10 +40,7 @@ public void testKeepFiles() throws Exception { .iterator(); Pack singlePack = packIt.next(); assertFalse(packIt.hasNext()); - String packFileName = singlePack.getPackFile().getPath(); - String keepFileName = packFileName.substring(0, - packFileName.lastIndexOf('.')) + ".keep"; - File keepFile = new File(keepFileName); + PackFile keepFile = singlePack.getPackFile().create(PackExt.KEEP); assertFalse(keepFile.exists()); assertTrue(keepFile.createNewFile()); bb.commit().add("A", "A2").add("B", "B2").create(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java new file mode 100644 index 000000000..619cfcac3 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.ObjectId; +import org.junit.Test; + +public class PackFileTest { + private static final ObjectId TEST_OID = ObjectId + .fromString("0123456789012345678901234567890123456789"); + + private static final String TEST_ID = TEST_OID.name(); + + private static final String PREFIX = "pack-"; + + private static final String OLD_PREFIX = "old-"; + + private static final String OLD_PACK = PREFIX + TEST_ID + "." + OLD_PREFIX + + PackExt.PACK.getExtension(); + + private static final File TEST_PACK_DIR = new File( + "/path/to/repo.git/objects/pack"); + + private static final File TEST_PRESERVED_DIR = new File(TEST_PACK_DIR, + "preserved"); + + private static final PackFile TEST_PACKFILE_NO_EXT = new PackFile( + new File(TEST_PACK_DIR, PREFIX + TEST_ID)); + + @Test + public void objectsAreSameFromAnyConstructor() throws Exception { + String name = PREFIX + TEST_ID + "." + PackExt.PACK.getExtension(); + File pack = new File(TEST_PACK_DIR, name); + PackFile pf = new PackFile(pack); + PackFile pfFromDirAndName = new PackFile(TEST_PACK_DIR, name); + assertPackFilesEqual(pf, pfFromDirAndName); + + PackFile pfFromOIdAndExt = new PackFile(TEST_PACK_DIR, TEST_OID, + PackExt.PACK); + assertPackFilesEqual(pf, pfFromOIdAndExt); + + PackFile pfFromIdAndExt = new PackFile(TEST_PACK_DIR, TEST_ID, + PackExt.PACK); + assertPackFilesEqual(pf, pfFromIdAndExt); + } + + @Test + public void idIsSameFromFileWithOrWithoutExt() throws Exception { + PackFile packWithExt = new PackFile(new File(TEST_PACK_DIR, + PREFIX + TEST_ID + "." + PackExt.PACK.getExtension())); + assertEquals(packWithExt.getId(), TEST_PACKFILE_NO_EXT.getId()); + } + + @Test + public void idIsSameFromFileWithOrWithoutPrefix() throws Exception { + PackFile packWithoutPrefix = new PackFile( + new File(TEST_PACK_DIR, TEST_ID)); + assertEquals(packWithoutPrefix.getId(), TEST_PACKFILE_NO_EXT.getId()); + } + + @Test + public void canCreatePreservedFromFile() throws Exception { + PackFile preserved = new PackFile( + new File(TEST_PRESERVED_DIR, OLD_PACK)); + assertTrue(preserved.getName().contains(OLD_PACK)); + assertEquals(preserved.getId(), TEST_ID); + assertEquals(preserved.getPackExt(), PackExt.PACK); + } + + @Test + public void canCreatePreservedFromDirAndName() throws Exception { + PackFile preserved = new PackFile(TEST_PRESERVED_DIR, OLD_PACK); + assertTrue(preserved.getName().contains(OLD_PACK)); + assertEquals(preserved.getId(), TEST_ID); + assertEquals(preserved.getPackExt(), PackExt.PACK); + } + + @Test + public void cannotCreatePreservedNoExtFromNonPreservedNoExt() + throws Exception { + assertThrows(IllegalArgumentException.class, () -> TEST_PACKFILE_NO_EXT + .createPreservedForDirectory(TEST_PRESERVED_DIR)); + } + + @Test + public void canCreateAnyExtFromAnyExt() throws Exception { + for (PackExt from : PackExt.values()) { + PackFile dotFrom = TEST_PACKFILE_NO_EXT.create(from); + for (PackExt to : PackExt.values()) { + PackFile dotTo = dotFrom.create(to); + File expected = new File(TEST_PACK_DIR, + PREFIX + TEST_ID + "." + to.getExtension()); + assertEquals(dotTo.getPackExt(), to); + assertEquals(dotFrom.getId(), dotTo.getId()); + assertEquals(expected.getName(), dotTo.getName()); + } + } + } + + @Test + public void canCreatePreservedFromAnyExt() throws Exception { + for (PackExt ext : PackExt.values()) { + PackFile nonPreserved = TEST_PACKFILE_NO_EXT.create(ext); + PackFile preserved = nonPreserved + .createPreservedForDirectory(TEST_PRESERVED_DIR); + File expected = new File(TEST_PRESERVED_DIR, + PREFIX + TEST_ID + "." + OLD_PREFIX + ext.getExtension()); + assertEquals(preserved.getName(), expected.getName()); + assertEquals(preserved.getId(), TEST_ID); + assertEquals(preserved.getPackExt(), nonPreserved.getPackExt()); + } + } + + @Test + public void canCreateAnyPreservedExtFromAnyPreservedExt() throws Exception { + // Preserved PackFiles must have an extension + PackFile preserved = new PackFile(TEST_PRESERVED_DIR, OLD_PACK); + for (PackExt from : PackExt.values()) { + PackFile preservedWithExt = preserved.create(from); + for (PackExt to : PackExt.values()) { + PackFile preservedNewExt = preservedWithExt.create(to); + File expected = new File(TEST_PRESERVED_DIR, PREFIX + TEST_ID + + "." + OLD_PREFIX + to.getExtension()); + assertEquals(preservedNewExt.getPackExt(), to); + assertEquals(preservedWithExt.getId(), preservedNewExt.getId()); + assertEquals(preservedNewExt.getName(), expected.getName()); + } + } + } + + @Test + public void canCreateNonPreservedFromAnyPreservedExt() throws Exception { + // Preserved PackFiles must have an extension + PackFile preserved = new PackFile(TEST_PRESERVED_DIR, OLD_PACK); + for (PackExt ext : PackExt.values()) { + PackFile preservedWithExt = preserved.create(ext); + PackFile nonPreserved = preservedWithExt + .createForDirectory(TEST_PACK_DIR); + File expected = new File(TEST_PACK_DIR, + PREFIX + TEST_ID + "." + ext.getExtension()); + assertEquals(nonPreserved.getName(), expected.getName()); + assertEquals(nonPreserved.getId(), TEST_ID); + assertEquals(nonPreserved.getPackExt(), + preservedWithExt.getPackExt()); + } + } + + private void assertPackFilesEqual(PackFile p1, PackFile p2) { + // for test purposes, considered equal if id, name, and ext are equal + assertEquals(p1.getId(), p2.getId()); + assertEquals(p1.getPackExt(), p2.getPackExt()); + assertEquals(p1.getName(), p2.getName()); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index 182e42265..a3596541f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -246,8 +246,8 @@ public void testDelta_FailsOver2GiB() throws Exception { File dir = new File(repo.getObjectDatabase().getDirectory(), "pack"); - File packName = new File(dir, idA.name() + ".pack"); - File idxName = new File(dir, idA.name() + ".idx"); + PackFile packName = new PackFile(dir, idA.name() + ".pack"); + PackFile idxName = packName.create(PackExt.INDEX); try (FileOutputStream f = new FileOutputStream(packName)) { f.write(packContents.toByteArray()); @@ -261,7 +261,7 @@ public void testDelta_FailsOver2GiB() throws Exception { new PackIndexWriterV1(f).write(list, footer); } - Pack pack = new Pack(packName, PackExt.INDEX.getBit()); + Pack pack = new Pack(packName, null); try { pack.get(wc, b); fail("expected LargeObjectException.ExceedsByteArrayLimit"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index 214ddb989..e422ab9db 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -34,6 +34,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRepository; @@ -305,9 +306,9 @@ public void testWritePack2DeltasReuseOffsets() throws IOException { @Test public void testWritePack2DeltasCRC32Copy() throws IOException { final File packDir = db.getObjectDatabase().getPackDirectory(); - final File crc32Pack = new File(packDir, + final PackFile crc32Pack = new PackFile(packDir, "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack"); - final File crc32Idx = new File(packDir, + final PackFile crc32Idx = new PackFile(packDir, "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx"); copyFile(JGitTestUtil.getTestResourceFile( "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"), @@ -471,10 +472,8 @@ public void testWriteIndex() throws Exception { config.setIndexVersion(2); writeVerifyPack4(false); - File packFile = pack.getPackFile(); - String name = packFile.getName(); - String base = name.substring(0, name.lastIndexOf('.')); - File indexFile = new File(packFile.getParentFile(), base + ".idx"); + PackFile packFile = pack.getPackFile(); + PackFile indexFile = packFile.create(PackExt.INDEX); // Validate that IndexPack came up with the right CRC32 value. final PackIndex idx1 = PackIndex.open(indexFile); @@ -685,14 +684,14 @@ private static PackIndex writePack(FileRepository repo, RevWalk walk, ObjectWalk ow = walk.toObjectWalkWithSameObjects(); pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE); - String id = pw.computeName().getName(); File packdir = repo.getObjectDatabase().getPackDirectory(); - File packFile = new File(packdir, "pack-" + id + ".pack"); + PackFile packFile = new PackFile(packdir, pw.computeName(), + PackExt.PACK); try (FileOutputStream packOS = new FileOutputStream(packFile)) { pw.writePack(NullProgressMonitor.INSTANCE, NullProgressMonitor.INSTANCE, packOS); } - File idxFile = new File(packdir, "pack-" + id + ".idx"); + PackFile idxFile = packFile.create(PackExt.INDEX); try (FileOutputStream idxOS = new FileOutputStream(idxFile)) { pw.writeIndex(idxOS); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/http/NetscapeCookieFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/http/NetscapeCookieFileTest.java index 6c8c3ba61..b11dd637f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/http/NetscapeCookieFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/transport/http/NetscapeCookieFileTest.java @@ -22,13 +22,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Arrays; -import java.util.Date; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.util.http.HttpCookiesMatcher; @@ -48,10 +48,14 @@ public class NetscapeCookieFileTest { private URL baseUrl; /** - * This is the expiration date that is used in the test cookie files + * This is the expiration date that is used in the test cookie files. */ - private static long JAN_01_2030_NOON = Instant - .parse("2030-01-01T12:00:00.000Z").toEpochMilli(); + private static final Instant TEST_EXPIRY_DATE = Instant + .parse("2030-01-01T12:00:00.000Z"); + + /** Earlier than TEST_EXPIRY_DATE. */ + private static final Instant TEST_DATE = TEST_EXPIRY_DATE.minus(180, + ChronoUnit.DAYS); @Before public void setUp() throws IOException { @@ -102,14 +106,13 @@ public void testWriteToNewFile() throws IOException { cookie.setPath("/"); cookie.setMaxAge(1000); cookies.add(cookie); - Date creationDate = new Date(); try (Writer writer = Files.newBufferedWriter(tmpFile, StandardCharsets.US_ASCII)) { - NetscapeCookieFile.write(writer, cookies, baseUrl, creationDate); + NetscapeCookieFile.write(writer, cookies, baseUrl, TEST_DATE); } String expectedExpiration = String - .valueOf(creationDate.getTime() + (cookie.getMaxAge() * 1000)); + .valueOf(TEST_DATE.getEpochSecond() + cookie.getMaxAge()); assertThat(Files.readAllLines(tmpFile, StandardCharsets.US_ASCII), CoreMatchers @@ -128,13 +131,12 @@ public void testWriteToExistingFile() throws IOException { HttpCookie cookie = new HttpCookie("key2", "value2"); cookie.setMaxAge(1000); cookies.add(cookie); - Date creationDate = new Date(); try (Writer writer = Files.newBufferedWriter(tmpFile, StandardCharsets.US_ASCII)) { - NetscapeCookieFile.write(writer, cookies, baseUrl, creationDate); + NetscapeCookieFile.write(writer, cookies, baseUrl, TEST_DATE); } String expectedExpiration = String - .valueOf(creationDate.getTime() + (cookie.getMaxAge() * 1000)); + .valueOf(TEST_DATE.getEpochSecond() + cookie.getMaxAge()); assertThat(Files.readAllLines(tmpFile, StandardCharsets.US_ASCII), CoreMatchers.equalTo( @@ -160,6 +162,21 @@ public void testWriteWhileSomeoneIsHoldingTheLock() } } + @Test + public void testReadCookieFileWithMilliseconds() throws IOException { + try (InputStream input = this.getClass() + .getResourceAsStream("cookies-with-milliseconds.txt")) { + Files.copy(input, tmpFile, StandardCopyOption.REPLACE_EXISTING); + } + NetscapeCookieFile cookieFile = new NetscapeCookieFile(tmpFile, + TEST_DATE); + long expectedMaxAge = Duration.between(TEST_DATE, TEST_EXPIRY_DATE) + .getSeconds(); + for (HttpCookie cookie : cookieFile.getCookies(true)) { + assertEquals(expectedMaxAge, cookie.getMaxAge()); + } + } + @Test public void testWriteAfterAnotherJgitProcessModifiedTheFile() throws IOException, InterruptedException { @@ -167,7 +184,8 @@ public void testWriteAfterAnotherJgitProcessModifiedTheFile() .getResourceAsStream("cookies-simple1.txt")) { Files.copy(input, tmpFile, StandardCopyOption.REPLACE_EXISTING); } - NetscapeCookieFile cookieFile = new NetscapeCookieFile(tmpFile); + NetscapeCookieFile cookieFile = new NetscapeCookieFile(tmpFile, + TEST_DATE); cookieFile.getCookies(true); // now modify file externally try (InputStream input = this.getClass() @@ -177,39 +195,19 @@ public void testWriteAfterAnotherJgitProcessModifiedTheFile() // now try to write cookieFile.write(baseUrl); - // validate that the external changes are there as well - // due to rounding errors (conversion from ms to sec to ms) - // the expiration date might not be exact List lines = Files.readAllLines(tmpFile, StandardCharsets.US_ASCII); assertEquals("Expected 3 lines", 3, lines.size()); - assertStringMatchesPatternWithInexactNumber(lines.get(0), - "some-domain1\tTRUE\t/some/path1\tFALSE\t(\\d*)\tkey1\tvalueFromSimple2", - JAN_01_2030_NOON, 1000); - assertStringMatchesPatternWithInexactNumber(lines.get(1), - "some-domain1\tTRUE\t/some/path1\tFALSE\t(\\d*)\tkey3\tvalueFromSimple2", - JAN_01_2030_NOON, 1000); - assertStringMatchesPatternWithInexactNumber(lines.get(2), - "some-domain1\tTRUE\t/some/path1\tFALSE\t(\\d*)\tkey2\tvalueFromSimple1", - JAN_01_2030_NOON, 1000); - } - - @SuppressWarnings("boxing") - private static final void assertStringMatchesPatternWithInexactNumber( - String string, String pattern, long expectedNumericValue, - long delta) { - java.util.regex.Matcher matcher = Pattern.compile(pattern) - .matcher(string); - assertTrue("Given string '" + string + "' does not match '" + pattern - + "'", matcher.matches()); - // extract numeric value - Long actualNumericValue = Long.decode(matcher.group(1)); - - assertTrue( - "Value is supposed to be close to " + expectedNumericValue - + " but is " + actualNumericValue + ".", - Math.abs(expectedNumericValue - actualNumericValue) <= delta); + assertEquals( + "some-domain1\tTRUE\t/some/path1\tFALSE\t1893499200\tkey1\tvalueFromSimple2", + lines.get(0)); + assertEquals( + "some-domain1\tTRUE\t/some/path1\tFALSE\t1893499200\tkey3\tvalueFromSimple2", + lines.get(1)); + assertEquals( + "some-domain1\tTRUE\t/some/path1\tFALSE\t1893499200\tkey2\tvalueFromSimple1", + lines.get(2)); } @Test @@ -229,14 +227,13 @@ public void testWriteAndReadCycle() throws IOException { cookie.setHttpOnly(true); cookies.add(cookie); - Date creationDate = new Date(); - try (Writer writer = Files.newBufferedWriter(tmpFile, StandardCharsets.US_ASCII)) { - NetscapeCookieFile.write(writer, cookies, baseUrl, creationDate); + NetscapeCookieFile.write(writer, cookies, baseUrl, TEST_DATE); } Set actualCookies = new NetscapeCookieFile(tmpFile, - creationDate).getCookies(true); + TEST_DATE) + .getCookies(true); assertThat(actualCookies, HttpCookiesMatcher.containsInOrder(cookies)); } @@ -246,15 +243,12 @@ public void testReadAndWriteCycle() throws IOException { .getResourceAsStream("cookies-simple1.txt")) { Files.copy(input, tmpFile, StandardCopyOption.REPLACE_EXISTING); } - // round up to the next second (to prevent rounding errors) - Date creationDate = new Date( - (System.currentTimeMillis() / 1000) * 1000); - Set cookies = new NetscapeCookieFile(tmpFile, creationDate) + Set cookies = new NetscapeCookieFile(tmpFile, TEST_DATE) .getCookies(true); Path tmpFile2 = folder.newFile().toPath(); try (Writer writer = Files.newBufferedWriter(tmpFile2, StandardCharsets.US_ASCII)) { - NetscapeCookieFile.write(writer, cookies, baseUrl, creationDate); + NetscapeCookieFile.write(writer, cookies, baseUrl, TEST_DATE); } // compare original file with newly written one, they should not differ assertEquals(Files.readAllLines(tmpFile), Files.readAllLines(tmpFile2)); @@ -267,13 +261,13 @@ public void testReadWithEmptyAndCommentLines() throws IOException { Files.copy(input, tmpFile, StandardCopyOption.REPLACE_EXISTING); } - Date creationDate = new Date(); Set cookies = new LinkedHashSet<>(); HttpCookie cookie = new HttpCookie("key2", "value2"); cookie.setDomain("some-domain2"); cookie.setPath("/some/path2"); - cookie.setMaxAge((JAN_01_2030_NOON - creationDate.getTime()) / 1000); + cookie.setMaxAge( + Duration.between(TEST_DATE, TEST_EXPIRY_DATE).getSeconds()); cookie.setSecure(true); cookie.setHttpOnly(true); cookies.add(cookie); @@ -281,11 +275,12 @@ public void testReadWithEmptyAndCommentLines() throws IOException { cookie = new HttpCookie("key3", "value3"); cookie.setDomain("some-domain3"); cookie.setPath("/some/path3"); - cookie.setMaxAge((JAN_01_2030_NOON - creationDate.getTime()) / 1000); + cookie.setMaxAge( + Duration.between(TEST_DATE, TEST_EXPIRY_DATE).getSeconds()); cookies.add(cookie); - Set actualCookies = new NetscapeCookieFile(tmpFile, creationDate) - .getCookies(true); + Set actualCookies = new NetscapeCookieFile(tmpFile, + TEST_DATE).getCookies(true); assertThat(actualCookies, HttpCookiesMatcher.containsInOrder(cookies)); } @@ -296,7 +291,7 @@ public void testReadInvalidFile() throws IOException { Files.copy(input, tmpFile, StandardCopyOption.REPLACE_EXISTING); } - new NetscapeCookieFile(tmpFile) - .getCookies(true); + assertTrue(new NetscapeCookieFile(tmpFile, TEST_DATE).getCookies(true) + .isEmpty()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkMergedIntoTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkMergedIntoTest.java index 2c21eb60d..2f16aa49e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkMergedIntoTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkMergedIntoTest.java @@ -11,6 +11,9 @@ import static org.junit.Assert.assertTrue; +import java.util.List; +import java.util.stream.Collectors; +import org.eclipse.jgit.lib.Ref; import org.junit.Test; public class RevWalkMergedIntoTest extends RevWalkTestCase { @@ -44,4 +47,82 @@ public void testOldCommitWalk() throws Exception { final RevCommit t = commit(n, o); assertTrue(rw.isMergedInto(b, t)); } + + @Test + public void testGetMergedInto() throws Exception { + /* + * i + * / \ + * A o + * / \ \ + * o1 o2 E + * / \ / \ + * B C D + */ + String b = "refs/heads/b"; + String c = "refs/heads/c"; + String d = "refs/heads/d"; + String e = "refs/heads/e"; + final RevCommit i = commit(); + final RevCommit a = commit(i); + final RevCommit o1 = commit(a); + final RevCommit o2 = commit(a); + createBranch(commit(o1), b); + createBranch(commit(o1, o2), c); + createBranch(commit(o2), d); + createBranch(commit(commit(i)), e); + + List modifiedResult = rw.getMergedInto(a, getRefs()) + .stream().map(Ref::getName).collect(Collectors.toList()); + + assertTrue(modifiedResult.size() == 3); + assertTrue(modifiedResult.contains(b)); + assertTrue(modifiedResult.contains(c)); + assertTrue(modifiedResult.contains(d)); + } + + @Test + public void testIsMergedIntoAny() throws Exception { + /* + * i + * / \ + * A o + * / \ + * o C + * / + * B + */ + String b = "refs/heads/b"; + String c = "refs/heads/c"; + final RevCommit i = commit(); + final RevCommit a = commit(i); + createBranch(commit(commit(a)), b); + createBranch(commit(commit(i)), c); + + assertTrue( rw.isMergedIntoAny(a, getRefs())); + } + + @Test + public void testIsMergedIntoAll() throws Exception { + /* + * + * A + * / \ + * o1 o2 + * / \ / \ + * B C D + */ + + String b = "refs/heads/b"; + String c = "refs/heads/c"; + String d = "refs/heads/c"; + final RevCommit a = commit(); + final RevCommit o1 = commit(a); + final RevCommit o2 = commit(a); + createBranch(commit(o1), b); + createBranch(commit(o1, o2), c); + createBranch(commit(o2), d); + + assertTrue(rw.isMergedIntoAll(a, getRefs())); + } } diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters deleted file mode 100644 index d389ac588..000000000 --- a/org.eclipse.jgit/.settings/.api_filters +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index c00203dd0..33087d762 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -139,6 +139,7 @@ configHandleMayBeLocked=config file handle may be locked by other process, {0}. connectionFailed=connection failed connectionTimeOut=Connection time out: {0} contextMustBeNonNegative=context must be >= 0 +cookieFilePathRelative=git config http.cookieFile contains a relative path, should be absolute: {0} corruptionDetectedReReadingAt=Corruption detected re-reading at {0} corruptObjectBadDate=bad date corruptObjectBadEmail=bad email @@ -743,6 +744,7 @@ unmergedPath=Unmerged path: {0} unmergedPaths=Repository contains unmerged paths unpackException=Exception while parsing pack stream unreadablePackIndex=Unreadable pack index: {0} +unrecognizedPackExtension=Unrecognized pack extension: {0} unrecognizedRef=Unrecognized ref: {0} unsetMark=Mark not set unsupportedAlternates=Alternates not supported diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java index 80e1b1829..75784c255 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java @@ -12,6 +12,7 @@ import static org.eclipse.jgit.diff.DiffEntry.Side.NEW; import static org.eclipse.jgit.diff.DiffEntry.Side.OLD; +import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BIG_FILE_THRESHOLD; import java.io.IOException; import java.util.ArrayList; @@ -97,6 +98,12 @@ private int sortOf(ChangeType changeType) { /** Limit in the number of files to consider for renames. */ private int renameLimit; + /** + * File size threshold (in bytes) for detecting renames. Files larger + * than this size will not be processed for renames. + */ + private int bigFileThreshold = DEFAULT_BIG_FILE_THRESHOLD; + /** Set if the number of adds or deletes was over the limit. */ private boolean overRenameLimit; @@ -208,6 +215,26 @@ public void setRenameLimit(int limit) { renameLimit = limit; } + /** + * Get file size threshold for detecting renames. Files larger + * than this size will not be processed for rename detection. + * + * @return threshold in bytes of the file size. + * @since 5.12 + */ + public int getBigFileThreshold() { return bigFileThreshold; } + + /** + * Set the file size threshold for detecting renames. Files larger than this + * threshold will be skipped during rename detection computation. + * + * @param threshold file size threshold in bytes. + * @since 5.12 + */ + public void setBigFileThreshold(int threshold) { + this.bigFileThreshold = threshold; + } + /** * Check if the detector is over the rename limit. *

@@ -493,6 +520,7 @@ private void findContentRenames(ContentSource.Pair reader, d = new SimilarityRenameDetector(reader, deleted, added); d.setRenameScore(getRenameScore()); + d.setBigFileThreshold(getBigFileThreshold()); d.compute(pm); overRenameLimit |= d.isTableOverflow(); deleted = d.getLeftOverSources(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java index 74a11a024..082f31d17 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java @@ -12,6 +12,7 @@ import static org.eclipse.jgit.diff.DiffEntry.Side.NEW; import static org.eclipse.jgit.diff.DiffEntry.Side.OLD; +import static org.eclipse.jgit.storage.pack.PackConfig.DEFAULT_BIG_FILE_THRESHOLD; import java.io.IOException; import java.util.ArrayList; @@ -80,6 +81,12 @@ class SimilarityRenameDetector { /** Score a pair must exceed to be considered a rename. */ private int renameScore = 60; + /** + * File size threshold (in bytes) for detecting renames. Files larger + * than this size will not be processed for renames. + */ + private int bigFileThreshold = DEFAULT_BIG_FILE_THRESHOLD; + /** Set if any {@link SimilarityIndex.TableFullException} occurs. */ private boolean tableOverflow; @@ -96,6 +103,10 @@ void setRenameScore(int score) { renameScore = score; } + void setBigFileThreshold(int threshold) { + bigFileThreshold = threshold; + } + void compute(ProgressMonitor pm) throws IOException, CancelledException { if (pm == null) pm = NullProgressMonitor.INSTANCE; @@ -253,6 +264,11 @@ private int buildMatrix(ProgressMonitor pm) continue; } + if (max > bigFileThreshold) { + pm.update(1); + continue; + } + if (s == null) { try { s = hash(OLD, srcEnt); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 9d215ca45..3eef49b1c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -167,6 +167,7 @@ public static JGitText get() { /***/ public String connectionFailed; /***/ public String connectionTimeOut; /***/ public String contextMustBeNonNegative; + /***/ public String cookieFilePathRelative; /***/ public String corruptionDetectedReReadingAt; /***/ public String corruptObjectBadDate; /***/ public String corruptObjectBadEmail; @@ -771,6 +772,7 @@ public static JGitText get() { /***/ public String unmergedPaths; /***/ public String unpackException; /***/ public String unreadablePackIndex; + /***/ public String unrecognizedPackExtension; /***/ public String unrecognizedRef; /***/ public String unsetMark; /***/ public String unsupportedAlternates; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 75de3be89..9ffff9f66 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -12,6 +12,8 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; import java.io.File; import java.io.FileOutputStream; @@ -346,7 +348,7 @@ private void deleteOldPacks(Collection oldPacks, if (shouldLoosen) { loosen(inserter, reader, oldPack, ids); } - prunePack(oldName); + prunePack(oldPack.getPackFile()); } } @@ -360,19 +362,17 @@ private void deleteOldPacks(Collection oldPacks, * moves the pack file to the preserved directory * * @param packFile - * @param packName - * @param ext * @param deleteOptions * @throws IOException */ - private void removeOldPack(File packFile, String packName, PackExt ext, - int deleteOptions) throws IOException { + private void removeOldPack(PackFile packFile, int deleteOptions) + throws IOException { if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); - String oldPackName = "pack-" + packName + ".old-" + ext.getExtension(); //$NON-NLS-1$ //$NON-NLS-2$ - File oldPackFile = new File(oldPackDir, oldPackName); + PackFile oldPackFile = packFile + .createPreservedForDirectory(oldPackDir); FileUtils.rename(packFile, oldPackFile); } else { FileUtils.delete(packFile, deleteOptions); @@ -401,27 +401,21 @@ private void prunePreserved() { * ".index" file and when failing to delete the ".pack" file we are left * with a ".pack" file without a ".index" file. * - * @param packName + * @param packFile */ - private void prunePack(String packName) { - PackExt[] extensions = PackExt.values(); + private void prunePack(PackFile packFile) { try { // Delete the .pack file first and if this fails give up on deleting // the other files int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING; - for (PackExt ext : extensions) - if (PackExt.PACK.equals(ext)) { - File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - removeOldPack(f, packName, ext, deleteOptions); - break; - } + removeOldPack(packFile.create(PackExt.PACK), deleteOptions); + // The .pack file has been deleted. Delete as many as the other // files as you can. deleteOptions |= FileUtils.IGNORE_ERRORS; - for (PackExt ext : extensions) { + for (PackExt ext : PackExt.values()) { if (!PackExt.PACK.equals(ext)) { - File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - removeOldPack(f, packName, ext, deleteOptions); + removeOldPack(packFile.create(ext), deleteOptions); } } } catch (IOException e) { @@ -973,20 +967,21 @@ private void deleteOrphans() { return; } - String base = null; + String latestId = null; for (String n : fileNames) { - if (n.endsWith(PACK_EXT) || n.endsWith(KEEP_EXT)) { - base = n.substring(0, n.lastIndexOf('.')); - } else { - if (base == null || !n.startsWith(base)) { - try { - Path delete = packDir.resolve(n); - FileUtils.delete(delete.toFile(), - FileUtils.RETRY | FileUtils.SKIP_MISSING); - LOG.warn(JGitText.get().deletedOrphanInPackDir, delete); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } + PackFile pf = new PackFile(packDir.toFile(), n); + PackExt ext = pf.getPackExt(); + if (ext.equals(PACK) || ext.equals(KEEP)) { + latestId = pf.getId(); + } + if (latestId == null || !pf.getId().equals(latestId)) { + // no pack or keep for this id + try { + FileUtils.delete(pf, + FileUtils.RETRY | FileUtils.SKIP_MISSING); + LOG.warn(JGitText.get().deletedOrphanInPackDir, pf); + } catch (IOException e) { + LOG.error(e.getMessage(), e); } } } @@ -1168,7 +1163,7 @@ private Pack writePack(@NonNull Set want, checkCancelled(); // create temporary files - String id = pw.computeName().getName(); + ObjectId id = pw.computeName(); File packdir = repo.getObjectDatabase().getPackDirectory(); packdir.mkdirs(); tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$ @@ -1218,7 +1213,8 @@ private Pack writePack(@NonNull Set want, } // rename the temporary files to real files - File realPack = nameFor(id, ".pack"); //$NON-NLS-1$ + File packDir = repo.getObjectDatabase().getPackDirectory(); + PackFile realPack = new PackFile(packDir, id, PackExt.PACK); repo.getObjectDatabase().closeAllPackHandles(realPack); tmpPack.setReadOnly(); @@ -1228,8 +1224,7 @@ private Pack writePack(@NonNull Set want, File tmpExt = tmpEntry.getValue(); tmpExt.setReadOnly(); - File realExt = nameFor(id, - "." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$ + PackFile realExt = new PackFile(packDir, id, tmpEntry.getKey()); try { FileUtils.rename(tmpExt, realExt, StandardCopyOption.ATOMIC_MOVE); @@ -1275,11 +1270,6 @@ private Pack writePack(@NonNull Set want, } } - private File nameFor(String name, String ext) { - File packdir = repo.getObjectDatabase().getPackDirectory(); - return new File(packdir, "pack-" + name + ext); //$NON-NLS-1$ - } - private void checkCancelled() throws CancelledException { if (pm.isCancelled() || Thread.currentThread().isInterrupted()) { throw new CancelledException(JGitText.get().operationCanceled); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index ae5bce698..f112947ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -17,6 +17,7 @@ import org.eclipse.jgit.internal.storage.pack.CachedPack; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation; @@ -88,6 +89,6 @@ private Pack getPackFile(String packName) throws FileNotFoundException { private String getPackFilePath(String packName) { final File packDir = odb.getPackDirectory(); - return new File(packDir, "pack-" + packName + ".pack").getPath(); //$NON-NLS-1$ //$NON-NLS-2$ + return new PackFile(packDir, packName, PackExt.PACK).getPath(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index e71a96060..627facca0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -11,8 +11,9 @@ package org.eclipse.jgit.internal.storage.file; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.BufferedReader; import java.io.File; @@ -79,7 +80,7 @@ public class ObjectDirectory extends FileObjectDatabase { private final PackDirectory packed; - private final File preservedDirectory; + private final PackDirectory preserved; private final File alternatesFile; @@ -117,10 +118,11 @@ public ObjectDirectory(final Config cfg, final File dir, objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ - preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ + File preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); loose = new LooseObjects(objects); packed = new PackDirectory(config, packDirectory); + preserved = new PackDirectory(config, preservedDirectory); this.fs = fs; this.shallowFile = shallowFile; @@ -156,7 +158,7 @@ public final File getPackDirectory() { * @return the location of the preserved directory. */ public final File getPreservedDirectory() { - return preservedDirectory; + return preserved.getDirectory(); } /** {@inheritDoc} */ @@ -216,26 +218,26 @@ public Collection getPacks() { * Add a single existing pack to the list of available pack files. */ @Override - public Pack openPack(File pack) - throws IOException { - final String p = pack.getName(); - if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ - throw new IOException(MessageFormat.format(JGitText.get().notAValidPack, pack)); - - // The pack and index are assumed to exist. The existence of other - // extensions needs to be explicitly checked. - // - int extensions = PACK.getBit() | INDEX.getBit(); - final String base = p.substring(0, p.length() - 4); - for (PackExt ext : PackExt.values()) { - if ((extensions & ext.getBit()) == 0) { - final String name = base + ext.getExtension(); - if (new File(pack.getParentFile(), name).exists()) - extensions |= ext.getBit(); - } + public Pack openPack(File pack) throws IOException { + PackFile pf; + try { + pf = new PackFile(pack); + } catch (IllegalArgumentException e) { + throw new IOException( + MessageFormat.format(JGitText.get().notAValidPack, pack), + e); } - Pack res = new Pack(pack, extensions); + String p = pf.getName(); + // TODO(nasserg): See if PackFile can do these checks instead + if (p.length() != 50 || !p.startsWith("pack-") //$NON-NLS-1$ + || !pf.getPackExt().equals(PACK)) { + throw new IOException( + MessageFormat.format(JGitText.get().notAValidPack, pack)); + } + + PackFile bitmapIdx = pf.create(BITMAP_INDEX); + Pack res = new Pack(pack, bitmapIdx.exists() ? bitmapIdx : null); packed.insert(res); return res; } @@ -250,7 +252,13 @@ public String toString() { @Override public boolean has(AnyObjectId objectId) { return loose.hasCached(objectId) - || hasPackedInSelfOrAlternate(objectId, null) + || hasPackedOrLooseInSelfOrAlternate(objectId) + || (restoreFromSelfOrAlternate(objectId, null) + && hasPackedOrLooseInSelfOrAlternate(objectId)); + } + + private boolean hasPackedOrLooseInSelfOrAlternate(AnyObjectId objectId) { + return hasPackedInSelfOrAlternate(objectId, null) || hasLooseInSelfOrAlternate(objectId, null); } @@ -319,6 +327,15 @@ private void resolve(Set matches, AbbreviatedObjectId id, @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { + ObjectLoader ldr = openObjectWithoutRestoring(curs, objectId); + if (ldr == null && restoreFromSelfOrAlternate(objectId, null)) { + ldr = openObjectWithoutRestoring(curs, objectId); + } + return ldr; + } + + private ObjectLoader openObjectWithoutRestoring(WindowCursor curs, AnyObjectId objectId) + throws IOException { if (loose.hasCached(objectId)) { ObjectLoader ldr = openLooseObject(curs, objectId); if (ldr != null) { @@ -380,8 +397,16 @@ ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id) } @Override - long getObjectSize(WindowCursor curs, AnyObjectId id) - throws IOException { + long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { + long sz = getObjectSizeWithoutRestoring(curs, id); + if (0 > sz && restoreFromSelfOrAlternate(id, null)) { + sz = getObjectSizeWithoutRestoring(curs, id); + } + return sz; + } + + private long getObjectSizeWithoutRestoring(WindowCursor curs, + AnyObjectId id) throws IOException { if (loose.hasCached(id)) { long len = loose.getSize(curs, id); if (0 <= len) { @@ -449,6 +474,51 @@ private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, } } + private boolean restoreFromSelfOrAlternate(AnyObjectId objectId, + Set skips) { + if (restoreFromSelf(objectId)) { + return true; + } + + skips = addMe(skips); + for (AlternateHandle alt : myAlternates()) { + if (!skips.contains(alt.getId())) { + if (alt.db.restoreFromSelfOrAlternate(objectId, skips)) { + return true; + } + } + } + return false; + } + + private boolean restoreFromSelf(AnyObjectId objectId) { + Pack preservedPack = preserved.getPack(objectId); + if (preservedPack == null) { + return false; + } + PackFile preservedFile = new PackFile(preservedPack.getPackFile()); + // Restore the index last since the set will be considered for use once + // the index appears. + for (PackExt ext : PackExt.values()) { + if (!INDEX.equals(ext)) { + restore(preservedFile.create(ext)); + } + } + restore(preservedFile.create(INDEX)); + return true; + } + + private boolean restore(PackFile preservedPack) { + PackFile restored = preservedPack + .createForDirectory(packed.getDirectory()); + try { + Files.createLink(restored.toPath(), preservedPack.toPath()); + } catch (IOException e) { + return false; + } + return true; + } + @Override InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 04d2ff8ab..dba8ccd99 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -27,6 +27,7 @@ import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -426,10 +427,10 @@ private PackLock renameAndOpenPack(String lockMessage) d.update(oeBytes); } - final String name = ObjectId.fromRaw(d.digest()).name(); - final File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$ - final File finalPack = new File(packDir, "pack-" + name + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ - final File finalIdx = new File(packDir, "pack-" + name + ".idx"); //$NON-NLS-1$ //$NON-NLS-2$ + ObjectId id = ObjectId.fromRaw(d.digest()); + File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$ + PackFile finalPack = new PackFile(packDir, id, PackExt.PACK); + PackFile finalIdx = finalPack.create(PackExt.INDEX); final PackLock keep = new PackLock(finalPack, db.getFS()); if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index d928633a7..5efd4c5bf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; @@ -38,6 +37,7 @@ import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; @@ -51,7 +51,6 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.BinaryDelta; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; @@ -78,13 +77,9 @@ public class Pack implements Iterable { public static final Comparator SORT = (a, b) -> b.packLastModified .compareTo(a.packLastModified); - private final File packFile; + private final PackFile packFile; - private final int extensions; - - private File keepFile; - - private volatile String packName; + private PackFile keepFile; final int hash; @@ -107,7 +102,8 @@ public class Pack implements Iterable { private volatile Exception invalidatingCause; - private boolean invalidBitmap; + @Nullable + private PackFile bitmapIdxFile; private AtomicInteger transientErrorCount = new AtomicInteger(); @@ -133,14 +129,14 @@ public class Pack implements Iterable { * * @param packFile * path of the .pack file holding the data. - * @param extensions - * additional pack file extensions with the same base as the pack + * @param bitmapIdxFile + * existing bitmap index file with the same base as the pack */ - public Pack(File packFile, int extensions) { - this.packFile = packFile; + public Pack(File packFile, @Nullable PackFile bitmapIdxFile) { + this.packFile = new PackFile(packFile); this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = fileSnapshot.lastModifiedInstant(); - this.extensions = extensions; + this.bitmapIdxFile = bitmapIdxFile; // Multiply by 31 here so we can more directly combine with another // value in WindowCache.hash(), without doing the multiply there. @@ -156,16 +152,18 @@ private PackIndex idx() throws IOException { idx = loadedIdx; if (idx == null) { if (invalid) { - throw new PackInvalidException(packFile, invalidatingCause); + throw new PackInvalidException(packFile, + invalidatingCause); } try { long start = System.currentTimeMillis(); - idx = PackIndex.open(extFile(INDEX)); + PackFile idxFile = packFile.create(INDEX); + idx = PackIndex.open(idxFile); if (LOG.isDebugEnabled()) { LOG.debug(String.format( "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ - extFile(INDEX).getAbsolutePath(), - Float.valueOf(extFile(INDEX).length() + idxFile.getAbsolutePath(), + Float.valueOf(idxFile.length() / (1024f * 1024)), Long.valueOf(System.currentTimeMillis() - start))); @@ -205,7 +203,7 @@ private PackIndex idx() throws IOException { * * @return the File object which locates this pack on disk. */ - public File getPackFile() { + public PackFile getPackFile() { return packFile; } @@ -225,16 +223,7 @@ public PackIndex getIndex() throws IOException { * @return name extracted from {@code pack-*.pack} pattern. */ public String getPackName() { - String name = packName; - if (name == null) { - name = getPackFile().getName(); - if (name.startsWith("pack-")) //$NON-NLS-1$ - name = name.substring("pack-".length()); //$NON-NLS-1$ - if (name.endsWith(".pack")) //$NON-NLS-1$ - name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$ - packName = name; - } - return name; + return packFile.getId(); } /** @@ -261,8 +250,9 @@ public boolean hasObject(AnyObjectId id) throws IOException { * @return true if a .keep file exist. */ public boolean shouldBeKept() { - if (keepFile == null) - keepFile = extFile(KEEP); + if (keepFile == null) { + keepFile = packFile.create(KEEP); + } return keepFile.exists(); } @@ -1132,26 +1122,28 @@ private long findEndOffset(long startOffset) } synchronized PackBitmapIndex getBitmapIndex() throws IOException { - if (invalid || invalidBitmap) + if (invalid || bitmapIdxFile == null) { return null; - if (bitmapIdx == null && hasExt(BITMAP_INDEX)) { + } + if (bitmapIdx == null) { final PackBitmapIndex idx; try { - idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(), + idx = PackBitmapIndex.open(bitmapIdxFile, idx(), getReverseIdx()); } catch (FileNotFoundException e) { // Once upon a time this bitmap file existed. Now it // has been removed. Most likely an external gc has // removed this packfile and the bitmap - invalidBitmap = true; - return null; + bitmapIdxFile = null; + return null; } // At this point, idx() will have set packChecksum. - if (Arrays.equals(packChecksum, idx.packChecksum)) + if (Arrays.equals(packChecksum, idx.packChecksum)) { bitmapIdx = idx; - else - invalidBitmap = true; + } else { + bitmapIdxFile = null; + } } return bitmapIdx; } @@ -1187,17 +1179,6 @@ private void setCorrupt(long offset) { } } - private File extFile(PackExt ext) { - String p = packFile.getName(); - int dot = p.lastIndexOf('.'); - String b = (dot < 0) ? p : p.substring(0, dot); - return new File(packFile.getParentFile(), b + '.' + ext.getExtension()); - } - - private boolean hasExt(PackExt ext) { - return (extensions & ext.getBit()) != 0; - } - @SuppressWarnings("nls") @Override public String toString() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index b2ba36bf9..73745d8c6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import java.io.File; @@ -20,13 +22,14 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.PackMismatchException; @@ -121,21 +124,36 @@ public String toString() { * * @param objectId * identity of the object to test for existence of. - * @return true if the specified object is stored in this PackDirectory. + * @return {@code true} if the specified object is stored in this PackDirectory. */ boolean has(AnyObjectId objectId) { + return getPack(objectId) != null; + } + + /** + * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} for the + * specified object if it is stored in this PackDirectory. + * + * @param objectId + * identity of the object to find the Pack for. + * @return {@link org.eclipse.jgit.internal.storage.file.Pack} which + * contains the specified object or {@code null} if it is not stored + * in this PackDirectory. + */ + @Nullable + Pack getPack(AnyObjectId objectId) { PackList pList; do { pList = packList.get(); for (Pack p : pList.packs) { try { if (p.hasObject(objectId)) { - return true; + return p; } } catch (IOException e) { - // The hasObject call should have only touched the index, - // so any failure here indicates the index is unreadable - // by this process, and the pack is likewise not readable. + // The hasObject call should have only touched the index, so + // any failure here indicates the index is unreadable by + // this process, and the pack is likewise not readable. LOG.warn(MessageFormat.format( JGitText.get().unableToReadPackfile, p.getPackFile().getAbsolutePath()), e); @@ -143,7 +161,7 @@ boolean has(AnyObjectId objectId) { } } } while (searchPacksAgain(pList)); - return false; + return null; } /** @@ -398,43 +416,29 @@ private PackList scanPacks(PackList original) { private PackList scanPacksImpl(PackList old) { final Map forReuse = reuseMap(old); final FileSnapshot snapshot = FileSnapshot.save(directory); - final Set names = listPackDirectory(); - final List list = new ArrayList<>(names.size() >> 2); + Map> packFilesByExtById = getPackFilesByExtById(); + List list = new ArrayList<>(packFilesByExtById.size()); boolean foundNew = false; - for (String indexName : names) { - // Must match "pack-[0-9a-f]{40}.idx" to be an index. - // - if (indexName.length() != 49 || !indexName.endsWith(".idx")) { //$NON-NLS-1$ - continue; - } - - final String base = indexName.substring(0, indexName.length() - 3); - int extensions = 0; - for (PackExt ext : PackExt.values()) { - if (names.contains(base + ext.getExtension())) { - extensions |= ext.getBit(); - } - } - - if ((extensions & PACK.getBit()) == 0) { + for (Map packFilesByExt : packFilesByExtById + .values()) { + PackFile packFile = packFilesByExt.get(PACK); + if (packFile == null || !packFilesByExt.containsKey(INDEX)) { // Sometimes C Git's HTTP fetch transport leaves a // .idx file behind and does not download the .pack. // We have to skip over such useless indexes. - // + // Also skip if we don't have any index for this id continue; } - final String packName = base + PACK.getExtension(); - final File packFile = new File(directory, packName); - final Pack oldPack = forReuse.get(packName); + Pack oldPack = forReuse.get(packFile.getName()); if (oldPack != null && !oldPack.getFileSnapshot().isModified(packFile)) { - forReuse.remove(packName); + forReuse.remove(packFile.getName()); list.add(oldPack); continue; } - list.add(new Pack(packFile, extensions)); + list.add(new Pack(packFile, packFilesByExt.get(BITMAP_INDEX))); foundNew = true; } @@ -487,18 +491,42 @@ private static Map reuseMap(PackList old) { return forReuse; } - private Set listPackDirectory() { + /** + * Scans the pack directory for + * {@link org.eclipse.jgit.internal.storage.file.PackFile}s and returns them + * organized by their extensions and their pack ids + * + * Skips files in the directory that we cannot create a + * {@link org.eclipse.jgit.internal.storage.file.PackFile} for. + * + * @return a map of {@link org.eclipse.jgit.internal.storage.file.PackFile}s + * and {@link org.eclipse.jgit.internal.storage.pack.PackExt}s keyed + * by pack ids + */ + private Map> getPackFilesByExtById() { final String[] nameList = directory.list(); if (nameList == null) { - return Collections.emptySet(); + return Collections.emptyMap(); } - final Set nameSet = new HashSet<>(nameList.length << 1); + Map> packFilesByExtById = new HashMap<>( + nameList.length / 2); // assume roughly 2 files per id for (String name : nameList) { - if (name.startsWith("pack-")) { //$NON-NLS-1$ - nameSet.add(name); + try { + PackFile pack = new PackFile(directory, name); + if (pack.getPackExt() != null) { + Map packByExt = packFilesByExtById + .get(pack.getId()); + if (packByExt == null) { + packByExt = new EnumMap<>(PackExt.class); + packFilesByExtById.put(pack.getId(), packByExt); + } + packByExt.put(pack.getPackExt(), pack); + } + } catch (IllegalArgumentException e) { + continue; } } - return nameSet; + return packFilesByExtById; } static final class PackList { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java new file mode 100644 index 000000000..19979d0ed --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import java.io.File; +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.ObjectId; + +/** + * A pack file (or pack related) File. + * + * Example: "pack-0123456789012345678901234567890123456789.idx" + */ +public class PackFile extends File { + private static final long serialVersionUID = 1L; + + private static final String PREFIX = "pack-"; //$NON-NLS-1$ + + private final String base; // PREFIX + id i.e. + // pack-0123456789012345678901234567890123456789 + + private final String id; // i.e. 0123456789012345678901234567890123456789 + + private final boolean hasOldPrefix; + + private final PackExt packExt; + + private static String createName(String id, PackExt extension) { + return PREFIX + id + '.' + extension.getExtension(); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param file + * File pointing to the location of the file. + */ + public PackFile(File file) { + this(file.getParentFile(), file.getName()); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param id + * the {@link ObjectId} for this pack + * @param ext + * the packExt of the name. + */ + public PackFile(File directory, ObjectId id, PackExt ext) { + this(directory, id.name(), ext); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param id + * the id (40 Hex char) section of the pack name. + * @param ext + * the packExt of the name. + */ + public PackFile(File directory, String id, PackExt ext) { + this(directory, createName(id, ext)); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param name + * Filename (last path section) of the PackFile + */ + public PackFile(File directory, String name) { + super(directory, name); + int dot = name.lastIndexOf('.'); + + if (dot < 0) { + base = name; + hasOldPrefix = false; + packExt = null; + } else { + base = name.substring(0, dot); + String tail = name.substring(dot + 1); // ["old-"] + extension + packExt = getPackExt(tail); + String old = tail.substring(0, + tail.length() - getExtension().length()); + hasOldPrefix = old.equals(getExtPrefix(true)); + } + + id = base.startsWith(PREFIX) ? base.substring(PREFIX.length()) : base; + } + + /** + * Getter for the field id. + * + * @return the id (40 Hex char) section of the name. + */ + public String getId() { + return id; + } + + /** + * Getter for the field packExt. + * + * @return the packExt of the name. + */ + public PackExt getPackExt() { + return packExt; + } + + /** + * Create a new similar PackFile with the given extension instead. + * + * @param ext + * PackExt the extension to use. + * @return a PackFile instance with specified extension + */ + public PackFile create(PackExt ext) { + return new PackFile(getParentFile(), getName(ext)); + } + + /** + * Create a new similar PackFile in the given directory. + * + * @param directory + * Directory to create the new PackFile in. + * @return a PackFile in the given directory + */ + public PackFile createForDirectory(File directory) { + return new PackFile(directory, getName(false)); + } + + /** + * Create a new similar preserved PackFile in the given directory. + * + * @param directory + * Directory to create the new PackFile in. + * @return a PackFile in the given directory with "old-" prefixing the + * extension + */ + public PackFile createPreservedForDirectory(File directory) { + return new PackFile(directory, getName(true)); + } + + private String getName(PackExt ext) { + return base + '.' + getExtPrefix(hasOldPrefix) + ext.getExtension(); + } + + private String getName(boolean isPreserved) { + return base + '.' + getExtPrefix(isPreserved) + getExtension(); + } + + private String getExtension() { + return packExt == null ? "" : packExt.getExtension(); //$NON-NLS-1$ + } + + private static String getExtPrefix(boolean isPreserved) { + return isPreserved ? "old-" : ""; //$NON-NLS-1$ //$NON-NLS-2$ + } + + private static PackExt getPackExt(String endsWithExtension) { + for (PackExt ext : PackExt.values()) { + if (endsWithExtension.endsWith(ext.getExtension())) { + return ext; + } + } + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().unrecognizedPackExtension, endsWithExtension)); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index a27a2b00c..d6209c4a7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -76,6 +76,7 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -273,16 +274,16 @@ public void flush() throws IOException { } Collections.sort(objectList); - File tmpIdx = idxFor(tmpPack); + File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - File realPack = new File(db.getPackDirectory(), - "pack-" + computeName(objectList).name() + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ + PackFile realPack = new PackFile(db.getPackDirectory(), + computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); tmpPack.setReadOnly(); FileUtils.rename(tmpPack, realPack, ATOMIC_MOVE); - File realIdx = idxFor(realPack); + PackFile realIdx = realPack.create(PackExt.INDEX); tmpIdx.setReadOnly(); try { FileUtils.rename(tmpIdx, realIdx, ATOMIC_MOVE); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java index bedc6939c..6fb775da8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java @@ -13,66 +13,26 @@ /** * A pack file extension. */ -public class PackExt { - private static volatile PackExt[] VALUES = new PackExt[] {}; - +public enum PackExt { /** A pack file extension. */ - public static final PackExt PACK = newPackExt("pack"); //$NON-NLS-1$ + PACK("pack"), //$NON-NLS-1$ /** A pack index file extension. */ - public static final PackExt INDEX = newPackExt("idx"); //$NON-NLS-1$ + INDEX("idx"), //$NON-NLS-1$ /** A keep pack file extension. */ - public static final PackExt KEEP = newPackExt("keep"); //$NON-NLS-1$ + KEEP("keep"), //$NON-NLS-1$ /** A pack bitmap index file extension. */ - public static final PackExt BITMAP_INDEX = newPackExt("bitmap"); //$NON-NLS-1$ + BITMAP_INDEX("bitmap"), //$NON-NLS-1$ /** A reftable file. */ - public static final PackExt REFTABLE = newPackExt("ref"); //$NON-NLS-1$ - - /** - * Get all of the PackExt values. - * - * @return all of the PackExt values. - */ - public static PackExt[] values() { - return VALUES; - } - - /** - * Returns a PackExt for the file extension and registers it in the values - * array. - * - * @param ext - * the file extension. - * @return the PackExt for the ext - */ - public static synchronized PackExt newPackExt(String ext) { - PackExt[] dst = new PackExt[VALUES.length + 1]; - for (int i = 0; i < VALUES.length; i++) { - PackExt packExt = VALUES[i]; - if (packExt.getExtension().equals(ext)) - return packExt; - dst[i] = packExt; - } - if (VALUES.length >= 32) - throw new IllegalStateException( - "maximum number of pack extensions exceeded"); //$NON-NLS-1$ - - PackExt value = new PackExt(ext, VALUES.length); - dst[VALUES.length] = value; - VALUES = dst; - return value; - } + REFTABLE("ref"); //$NON-NLS-1$ private final String ext; - private final int pos; - - private PackExt(String ext, int pos) { + private PackExt(String ext) { this.ext = ext; - this.pos = pos; } /** @@ -85,12 +45,12 @@ public String getExtension() { } /** - * Get the position of the extension in the values array. + * Get the position of the extension in the enum declaration. * - * @return the position of the extension in the values array. + * @return the position of the extension in the enum declaration. */ public int getPosition() { - return pos; + return ordinal(); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/http/NetscapeCookieFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/http/NetscapeCookieFile.java index 49f26c788..dae717305 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/http/NetscapeCookieFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/http/NetscapeCookieFile.java @@ -22,11 +22,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.text.MessageFormat; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; -import java.util.Date; import java.util.LinkedHashSet; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; @@ -53,6 +54,7 @@ * In general this class is not thread-safe. So any consumer needs to take care * of synchronization! * + * @see Cookie file format * @see Netscape Cookie File * Format * @see cookies = null; @@ -104,13 +106,13 @@ public final class NetscapeCookieFile { * where to find the cookie file */ public NetscapeCookieFile(Path path) { - this(path, new Date()); + this(path, Instant.now()); } - NetscapeCookieFile(Path path, Date creationDate) { + NetscapeCookieFile(Path path, Instant createdAt) { this.path = path; this.snapshot = FileSnapshot.DIRTY; - this.creationDate = creationDate; + this.createdAt = createdAt; } /** @@ -142,7 +144,7 @@ public Set getCookies(boolean refresh) { if (cookies == null || refresh) { try { byte[] in = getFileContentIfModified(); - Set newCookies = parseCookieFile(in, creationDate); + Set newCookies = parseCookieFile(in, createdAt); if (cookies != null) { cookies = mergeCookies(newCookies, cookies); } else { @@ -168,9 +170,9 @@ public Set getCookies(boolean refresh) { * * @param input * the file content to parse - * @param creationDate - * the date for the creation of the cookies (used to calculate - * the maxAge based on the expiration date given within the file) + * @param createdAt + * cookie creation time; used to calculate the maxAge based on + * the expiration date given within the file * @return the set of parsed cookies from the given file (even expired * ones). If there is more than one cookie with the same name in * this file the last one overwrites the first one! @@ -180,7 +182,7 @@ public Set getCookies(boolean refresh) { * if the given file does not have a proper format */ private static Set parseCookieFile(@NonNull byte[] input, - @NonNull Date creationDate) + @NonNull Instant createdAt) throws IOException, IllegalArgumentException { String decoded = RawParseUtils.decode(StandardCharsets.US_ASCII, input); @@ -190,7 +192,7 @@ private static Set parseCookieFile(@NonNull byte[] input, new StringReader(decoded))) { String line; while ((line = reader.readLine()) != null) { - HttpCookie cookie = parseLine(line, creationDate); + HttpCookie cookie = parseLine(line, createdAt); if (cookie != null) { cookies.add(cookie); } @@ -200,7 +202,7 @@ private static Set parseCookieFile(@NonNull byte[] input, } private static HttpCookie parseLine(@NonNull String line, - @NonNull Date creationDate) { + @NonNull Instant createdAt) { if (line.isEmpty() || (line.startsWith("#") //$NON-NLS-1$ && !line.startsWith(HTTP_ONLY_PREAMBLE))) { return null; @@ -236,7 +238,12 @@ private static HttpCookie parseLine(@NonNull String line, cookie.setSecure(Boolean.parseBoolean(cookieLineParts[3])); long expires = Long.parseLong(cookieLineParts[4]); - long maxAge = (expires - creationDate.getTime()) / 1000; + // Older versions stored milliseconds. This heuristic to detect that + // will cause trouble in the year 33658. :-) + if (cookieLineParts[4].length() == 13) { + expires = TimeUnit.MILLISECONDS.toSeconds(expires); + } + long maxAge = expires - createdAt.getEpochSecond(); if (maxAge <= 0) { return null; // skip expired cookies } @@ -245,7 +252,7 @@ private static HttpCookie parseLine(@NonNull String line, } /** - * Read the underying file and return its content but only in case it has + * Read the underlying file and return its content but only in case it has * been modified since the last access. *

* Internally calculates the hash and maintains {@link FileSnapshot}s to @@ -333,7 +340,7 @@ public void write(URL url) throws IOException, InterruptedException { path); // reread new changes if necessary Set cookiesFromFile = NetscapeCookieFile - .parseCookieFile(cookieFileContent, creationDate); + .parseCookieFile(cookieFileContent, createdAt); this.cookies = mergeCookies(cookiesFromFile, cookies); } } catch (FileNotFoundException e) { @@ -343,7 +350,7 @@ public void write(URL url) throws IOException, InterruptedException { ByteArrayOutputStream output = new ByteArrayOutputStream(); try (Writer writer = new OutputStreamWriter(output, StandardCharsets.US_ASCII)) { - write(writer, cookies, url, creationDate); + write(writer, cookies, url, createdAt); } LockFile lockFile = new LockFile(path.toFile()); for (int retryCount = 0; retryCount < LOCK_ACQUIRE_MAX_RETRY_COUNT; retryCount++) { @@ -377,24 +384,23 @@ public void write(URL url) throws IOException, InterruptedException { * @param url * the url for which to write the cookie (to derive the default * values for certain cookie attributes) - * @param creationDate - * the date when the cookie has been created. Important for - * calculation the cookie expiration time (calculated from - * cookie's maxAge and this creation time) + * @param createdAt + * cookie creation time; used to calculate a cookie's expiration + * time * @throws IOException * if an I/O error occurs */ static void write(@NonNull Writer writer, @NonNull Collection cookies, @NonNull URL url, - @NonNull Date creationDate) throws IOException { + @NonNull Instant createdAt) throws IOException { for (HttpCookie cookie : cookies) { - writeCookie(writer, cookie, url, creationDate); + writeCookie(writer, cookie, url, createdAt); } } private static void writeCookie(@NonNull Writer writer, @NonNull HttpCookie cookie, @NonNull URL url, - @NonNull Date creationDate) throws IOException { + @NonNull Instant createdAt) throws IOException { if (cookie.getMaxAge() <= 0) { return; // skip expired cookies } @@ -422,7 +428,7 @@ private static void writeCookie(@NonNull Writer writer, final String expirationDate; // whenCreated field is not accessible in HttpCookie expirationDate = String - .valueOf(creationDate.getTime() + (cookie.getMaxAge() * 1000)); + .valueOf(createdAt.getEpochSecond() + cookie.getMaxAge()); writer.write(expirationDate); writer.write(COLUMN_SEPARATOR); writer.write(cookie.getName()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index 631d861c0..5d5ba12ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -32,10 +32,13 @@ import org.eclipse.jgit.lib.AsyncObjectLoaderQueue; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.MutableObjectId; +import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.treewalk.filter.TreeFilter; @@ -181,6 +184,12 @@ public class RevWalk implements Iterable, AutoCloseable { boolean shallowCommitsInitialized; + private enum GetMergedIntoStrategy { + RETURN_ON_FIRST_FOUND, + RETURN_ON_FIRST_NOT_FOUND, + EVALUATE_ALL + } + /** * Create a new revision walker for a given repository. * @@ -424,6 +433,145 @@ public boolean isMergedInto(RevCommit base, RevCommit tip) } } + /** + * Determine the Refs into which a commit is merged. + *

+ * A commit is merged into a ref if we can find a path of commits that leads + * from that specific ref and ends at commit. + *

+ * + * @param commit + * commit the caller thinks is reachable from refs. + * @param refs + * refs to start iteration from, and which is most likely a + * descendant (child) of commit. + * @return list of refs that are reachable from commit. + * @throws java.io.IOException + * a pack file or loose object could not be read. + * @since 5.12 + */ + public List getMergedInto(RevCommit commit, Collection refs) + throws IOException{ + return getMergedInto(commit, refs, NullProgressMonitor.INSTANCE); + } + + /** + * Determine the Refs into which a commit is merged. + *

+ * A commit is merged into a ref if we can find a path of commits that leads + * from that specific ref and ends at commit. + *

+ * + * @param commit + * commit the caller thinks is reachable from refs. + * @param refs + * refs to start iteration from, and which is most likely a + * descendant (child) of commit. + * @param monitor + * the callback for progress and cancellation + * @return list of refs that are reachable from commit. + * @throws java.io.IOException + * a pack file or loose object could not be read. + * @since 5.12 + */ + public List getMergedInto(RevCommit commit, Collection refs, + ProgressMonitor monitor) throws IOException{ + return getMergedInto(commit, refs, + GetMergedIntoStrategy.EVALUATE_ALL, + monitor); + } + + /** + * Determine if a commit is merged into any of the given + * refs. + * + * @param commit + * commit the caller thinks is reachable from refs. + * @param refs + * refs to start iteration from, and which is most likely a + * descendant (child) of commit. + * @return true if commit is merged into any of the refs; false otherwise. + * @throws java.io.IOException + * a pack file or loose object could not be read. + * @since 5.12 + */ + public boolean isMergedIntoAny(RevCommit commit, Collection refs) + throws IOException { + return getMergedInto(commit, refs, + GetMergedIntoStrategy.RETURN_ON_FIRST_FOUND, + NullProgressMonitor.INSTANCE).size() > 0; + } + + /** + * Determine if a commit is merged into all of the given + * refs. + * + * @param commit + * commit the caller thinks is reachable from refs. + * @param refs + * refs to start iteration from, and which is most likely a + * descendant (child) of commit. + * @return true if commit is merged into all of the refs; false otherwise. + * @throws java.io.IOException + * a pack file or loose object could not be read. + * @since 5.12 + */ + public boolean isMergedIntoAll(RevCommit commit, Collection refs) + throws IOException { + return getMergedInto(commit, refs, + GetMergedIntoStrategy.RETURN_ON_FIRST_NOT_FOUND, + NullProgressMonitor.INSTANCE).size() + == refs.size(); + } + + private List getMergedInto(RevCommit needle, Collection haystacks, + Enum returnStrategy, ProgressMonitor monitor) throws IOException { + List result = new ArrayList<>(); + RevFilter oldRF = filter; + TreeFilter oldTF = treeFilter; + try { + finishDelayedFreeFlags(); + filter = RevFilter.ALL; + treeFilter = TreeFilter.ALL; + for (Ref r: haystacks) { + if (monitor.isCancelled()) { + return result; + } + monitor.update(1); + RevObject o = parseAny(r.getObjectId()); + if (!(o instanceof RevCommit)) { + continue; + } + RevCommit c = (RevCommit) o; + resetRetain(RevFlag.UNINTERESTING); + markStart(c); + boolean commitFound = false; + RevCommit next; + while ((next = next()) != null) { + if (References.isSameObject(next, needle)) { + result.add(r); + if (returnStrategy == GetMergedIntoStrategy.RETURN_ON_FIRST_FOUND) { + return result; + } + commitFound = true; + break; + } + } + if(!commitFound){ + markUninteresting(c); + if (returnStrategy == GetMergedIntoStrategy.RETURN_ON_FIRST_NOT_FOUND) { + return result; + } + } + } + } finally { + reset(~freeFlags & APP_FLAGS); + filter = oldRF; + treeFilter = oldTF; + } + return result; + } + /** * Pop the next most recent commit. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalkUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalkUtils.java index 3feb9c5a4..e52e91631 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalkUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalkUtils.java @@ -159,15 +159,12 @@ public static List findBranchesReachableFrom(RevCommit commit, // Make sure commit is from the same RevWalk commit = revWalk.parseCommit(commit.getId()); revWalk.reset(); - List result = new ArrayList<>(); + List filteredRefs = new ArrayList<>(); monitor.beginTask(JGitText.get().searchForReachableBranches, refs.size()); final int SKEW = 24*3600; // one day clock skew for (Ref ref : refs) { - if (monitor.isCancelled()) - return result; - monitor.update(1); RevObject maybehead = revWalk.parseAny(ref.getObjectId()); if (!(maybehead instanceof RevCommit)) continue; @@ -179,9 +176,9 @@ public static List findBranchesReachableFrom(RevCommit commit, if (headCommit.getCommitTime() + SKEW < commit.getCommitTime()) continue; - if (revWalk.isMergedInto(commit, headCommit)) - result.add(ref); + filteredRefs.add(ref); } + List result = revWalk.getMergedInto(commit, filteredRefs, monitor); monitor.endTask(); return result; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 2e5d18dc1..0710d3fdf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -35,6 +35,7 @@ import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -53,8 +54,6 @@ import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.cert.CertPathBuilderException; import java.security.cert.CertPathValidatorException; @@ -101,6 +100,7 @@ import org.eclipse.jgit.transport.http.HttpConnection; import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.eclipse.jgit.transport.http.HttpConnectionFactory2; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.HttpSupport; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; @@ -1157,17 +1157,28 @@ IOException wrongContentType(String expType, String actType) { return new TransportException(uri, why); } - private static NetscapeCookieFile getCookieFileFromConfig( + private NetscapeCookieFile getCookieFileFromConfig( HttpConfig config) { - if (!StringUtils.isEmptyOrNull(config.getCookieFile())) { + String path = config.getCookieFile(); + if (!StringUtils.isEmptyOrNull(path)) { try { - Path cookieFilePath = Paths.get(config.getCookieFile()); + FS fs = local != null ? local.getFS() : FS.DETECTED; + File f; + if (path.startsWith("~/")) { //$NON-NLS-1$ + f = fs.resolve(fs.userHome(), path.substring(2)); + } else { + f = new File(path); + if (!f.isAbsolute()) { + f = fs.resolve(null, path); + LOG.warn(MessageFormat.format( + JGitText.get().cookieFilePathRelative, f)); + } + } return NetscapeCookieFileCache.getInstance(config) - .getEntry(cookieFilePath); + .getEntry(f.toPath()); } catch (InvalidPathException e) { LOG.warn(MessageFormat.format( - JGitText.get().couldNotReadCookieFile, - config.getCookieFile()), e); + JGitText.get().couldNotReadCookieFile, path), e); } } return null; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java index f2eac8d24..03ef852c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java @@ -13,6 +13,7 @@ import static org.eclipse.jgit.transport.WalkRemoteObjectDatabase.ROOT_DIR; import java.io.BufferedOutputStream; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; @@ -26,6 +27,8 @@ import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -189,9 +192,8 @@ public void close() { private void sendpack(final List updates, final ProgressMonitor monitor) throws TransportException { - String pathPack = null; - String pathIdx = null; - + PackFile pack = null; + PackFile idx = null; try (PackWriter writer = new PackWriter(transport.getPackConfig(), local.newObjectReader())) { @@ -217,31 +219,33 @@ private void sendpack(final List updates, for (String n : dest.getPackNames()) packNames.put(n, n); - final String base = "pack-" + writer.computeName().name(); //$NON-NLS-1$ - final String packName = base + ".pack"; //$NON-NLS-1$ - pathPack = "pack/" + packName; //$NON-NLS-1$ - pathIdx = "pack/" + base + ".idx"; //$NON-NLS-1$ //$NON-NLS-2$ + File packDir = new File("pack"); //$NON-NLS-1$ + pack = new PackFile(packDir, writer.computeName(), + PackExt.PACK); + idx = pack.create(PackExt.INDEX); - if (packNames.remove(packName) != null) { + if (packNames.remove(pack.getName()) != null) { // The remote already contains this pack. We should // remove the index before overwriting to prevent bad // offsets from appearing to clients. // dest.writeInfoPacks(packNames.keySet()); - dest.deleteFile(pathIdx); + dest.deleteFile(idx.getPath()); } // Write the pack file, then the index, as readers look the // other direction (index, then pack file). // - String wt = "Put " + base.substring(0, 12); //$NON-NLS-1$ + String wt = "Put " + pack.getName().substring(0, 12); //$NON-NLS-1$ try (OutputStream os = new BufferedOutputStream( - dest.writeFile(pathPack, monitor, wt + "..pack"))) { //$NON-NLS-1$ + dest.writeFile(pack.getPath(), monitor, + wt + "." + pack.getPackExt().getExtension()))) { //$NON-NLS-1$ writer.writePack(monitor, monitor, os); } try (OutputStream os = new BufferedOutputStream( - dest.writeFile(pathIdx, monitor, wt + "..idx"))) { //$NON-NLS-1$ + dest.writeFile(idx.getPath(), monitor, + wt + "." + idx.getPackExt().getExtension()))) { //$NON-NLS-1$ writer.writeIndex(os); } @@ -250,22 +254,22 @@ private void sendpack(final List updates, // and discover the most recent objects there. // final ArrayList infoPacks = new ArrayList<>(); - infoPacks.add(packName); + infoPacks.add(pack.getName()); infoPacks.addAll(packNames.keySet()); dest.writeInfoPacks(infoPacks); } catch (IOException err) { - safeDelete(pathIdx); - safeDelete(pathPack); + safeDelete(idx); + safeDelete(pack); throw new TransportException(uri, JGitText.get().cannotStoreObjects, err); } } - private void safeDelete(String path) { + private void safeDelete(File path) { if (path != null) { try { - dest.deleteFile(path); + dest.deleteFile(path.getPath()); } catch (IOException cleanupFailure) { // Ignore the deletion failure. We probably are // already failing and were just trying to pick