From 971dafd302ae4bb2d69345f9624774ff226feebd Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Wed, 10 Feb 2021 22:51:05 -0700 Subject: [PATCH 01/11] Create a PackFile class for Pack filenames The PackFile class is intended to be a central place to do all common pack filename manipulation and parsing to help reduce repeated code and bugs. Use the PackFile class in the Pack class and in many tests to ensure it works well in a variety of situations. Later changes will expand use of PackFiles to even more areas. Change-Id: I921b30f865759162bae46ddd2c6d669de06add4a Signed-off-by: Nasser Grainawi Signed-off-by: Matthias Sohn --- .../eclipse/jgit/junit/TestRepository.java | 11 +- .../storage/file/ConcurrentRepackTest.java | 9 +- .../storage/file/GcBasicPackingTest.java | 11 +- .../storage/file/GcKeepFilesTest.java | 7 +- .../internal/storage/file/PackFileTest.java | 150 +++++++++++++++++ .../jgit/internal/storage/file/PackTest.java | 4 +- .../internal/storage/file/PackWriterTest.java | 15 +- .../eclipse/jgit/internal/JGitText.properties | 1 + .../org/eclipse/jgit/internal/JGitText.java | 1 + .../jgit/internal/storage/file/Pack.java | 45 ++--- .../jgit/internal/storage/file/PackFile.java | 154 ++++++++++++++++++ 11 files changed, 348 insertions(+), 60 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index e3eb2c536..24f7741f1 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java @@ -44,7 +44,9 @@ import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.internal.storage.file.ObjectDirectory; import org.eclipse.jgit.internal.storage.file.Pack; +import org.eclipse.jgit.internal.storage.file.PackFile; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -906,7 +908,7 @@ public void packAndPrune() throws Exception { ObjectDirectory odb = (ObjectDirectory) db.getObjectDatabase(); NullProgressMonitor m = NullProgressMonitor.INSTANCE; - final File pack, idx; + final PackFile pack, idx; try (PackWriter pw = new PackWriter(db)) { Set all = new HashSet<>(); for (Ref r : db.getRefDatabase().getRefs()) @@ -922,7 +924,7 @@ public void packAndPrune() throws Exception { } pack.setReadOnly(); - idx = nameFor(odb, name, ".idx"); + idx = pack.create(PackExt.INDEX); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(idx))) { pw.writeIndex(out); @@ -960,9 +962,10 @@ private static void prunePacked(ObjectDirectory odb) throws IOException { } } - private static File nameFor(ObjectDirectory odb, ObjectId name, String t) { + private static PackFile nameFor(ObjectDirectory odb, ObjectId name, + String t) { File packdir = odb.getPackDirectory(); - return new File(packdir, "pack-" + name.name() + t); + return new PackFile(packdir, "pack-" + name.name() + t); } private void writeFile(File p, byte[] bin) throws IOException, diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java index da3b5bbc3..ca6350735 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java @@ -27,6 +27,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.AnyObjectId; @@ -194,8 +195,8 @@ private File[] pack(Repository src, RevObject... list) } final ObjectId name = pw.computeName(); - final File packFile = fullPackFileName(name, ".pack"); - final File idxFile = fullPackFileName(name, ".idx"); + final PackFile packFile = fullPackFileName(name, ".pack"); + final PackFile idxFile = packFile.create(PackExt.INDEX); final File[] files = new File[] { packFile, idxFile }; write(files, pw); return files; @@ -242,9 +243,9 @@ private static void touch(Instant begin, File dir) throws IOException { } } - private File fullPackFileName(ObjectId name, String suffix) { + private PackFile fullPackFileName(ObjectId name, String suffix) { final File packdir = db.getObjectDatabase().getPackDirectory(); - return new File(packdir, "pack-" + name.name() + suffix); + return new PackFile(packdir, "pack-" + name.name() + suffix); } private RevObject writeBlob(Repository repo, String data) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index 42e423845..61538c9c2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -295,7 +295,7 @@ private void testPreserveOldPacks() throws Exception { // pack loose object into packfile gc.setExpireAgeMillis(0); gc.gc(); - File oldPackfile = tr.getRepository().getObjectDatabase().getPacks() + PackFile oldPackfile = tr.getRepository().getObjectDatabase().getPacks() .iterator().next().getPackFile(); assertTrue(oldPackfile.exists()); @@ -309,12 +309,9 @@ private void testPreserveOldPacks() throws Exception { configureGc(gc, false).setPreserveOldPacks(true); gc.gc(); - File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); - String oldPackFileName = oldPackfile.getName(); - String oldPackName = oldPackFileName.substring(0, - oldPackFileName.lastIndexOf('.')) + ".old-pack"; //$NON-NLS-1$ - File preservePackFile = new File(oldPackDir, oldPackName); - assertTrue(preservePackFile.exists()); + File preservedPackFile = oldPackfile.createPreservedForDirectory( + repo.getObjectDatabase().getPreservedDirectory()); + assertTrue(preservedPackFile.exists()); } private PackConfig configureGc(GC myGc, boolean aggressive) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java index 8472983d5..5fcdd3757 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java @@ -14,10 +14,10 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.File; import java.util.Iterator; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.junit.Test; @@ -40,10 +40,7 @@ public void testKeepFiles() throws Exception { .iterator(); Pack singlePack = packIt.next(); assertFalse(packIt.hasNext()); - String packFileName = singlePack.getPackFile().getPath(); - String keepFileName = packFileName.substring(0, - packFileName.lastIndexOf('.')) + ".keep"; - File keepFile = new File(keepFileName); + PackFile keepFile = singlePack.getPackFile().create(PackExt.KEEP); assertFalse(keepFile.exists()); assertTrue(keepFile.createNewFile()); bb.commit().add("A", "A2").add("B", "B2").create(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java new file mode 100644 index 000000000..88d25b73d --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.junit.Test; + +public class PackFileTest { + private static final String TEST_ID = "0123456789012345678901234567890123456789"; + + private static final String PREFIX = "pack-"; + + private static final String OLD_PREFIX = "old-"; + + private static final String OLD_PACK = PREFIX + TEST_ID + "." + OLD_PREFIX + + PackExt.PACK.getExtension(); + + private static final File TEST_PACK_DIR = new File( + "/path/to/repo.git/objects/pack"); + + private static final File TEST_PRESERVED_DIR = new File(TEST_PACK_DIR, + "preserved"); + + private static final PackFile TEST_PACKFILE_NO_EXT = new PackFile( + new File(TEST_PACK_DIR, PREFIX + TEST_ID)); + + @Test + public void idIsSameFromFileOrDirAndName() throws Exception { + File pack = new File(TEST_PACK_DIR, PREFIX + TEST_ID); + PackFile pf = new PackFile(pack); + PackFile pfFromDirAndName = new PackFile(TEST_PACK_DIR, + PREFIX + TEST_ID); + assertEquals(pf.getId(), pfFromDirAndName.getId()); + } + + @Test + public void idIsSameFromFileWithOrWithoutExt() throws Exception { + PackFile packWithExt = new PackFile(new File(TEST_PACK_DIR, + PREFIX + TEST_ID + "." + PackExt.PACK.getExtension())); + assertEquals(packWithExt.getId(), TEST_PACKFILE_NO_EXT.getId()); + } + + @Test + public void idIsSameFromFileWithOrWithoutPrefix() throws Exception { + PackFile packWithoutPrefix = new PackFile( + new File(TEST_PACK_DIR, TEST_ID)); + assertEquals(packWithoutPrefix.getId(), TEST_PACKFILE_NO_EXT.getId()); + } + + @Test + public void canCreatePreservedFromFile() throws Exception { + PackFile preserved = new PackFile( + new File(TEST_PRESERVED_DIR, OLD_PACK)); + assertTrue(preserved.getName().contains(OLD_PACK)); + assertEquals(preserved.getId(), TEST_ID); + assertEquals(preserved.getPackExt(), PackExt.PACK); + } + + @Test + public void canCreatePreservedFromDirAndName() throws Exception { + PackFile preserved = new PackFile(TEST_PRESERVED_DIR, OLD_PACK); + assertTrue(preserved.getName().contains(OLD_PACK)); + assertEquals(preserved.getId(), TEST_ID); + assertEquals(preserved.getPackExt(), PackExt.PACK); + } + + @Test + public void cannotCreatePreservedNoExtFromNonPreservedNoExt() + throws Exception { + assertThrows(IllegalArgumentException.class, () -> TEST_PACKFILE_NO_EXT + .createPreservedForDirectory(TEST_PRESERVED_DIR)); + } + + @Test + public void canCreateAnyExtFromAnyExt() throws Exception { + for (PackExt from : PackExt.values()) { + PackFile dotFrom = TEST_PACKFILE_NO_EXT.create(from); + for (PackExt to : PackExt.values()) { + PackFile dotTo = dotFrom.create(to); + File expected = new File(TEST_PACK_DIR, + PREFIX + TEST_ID + "." + to.getExtension()); + assertEquals(dotTo.getPackExt(), to); + assertEquals(dotFrom.getId(), dotTo.getId()); + assertEquals(expected.getName(), dotTo.getName()); + } + } + } + + @Test + public void canCreatePreservedFromAnyExt() throws Exception { + for (PackExt ext : PackExt.values()) { + PackFile nonPreserved = TEST_PACKFILE_NO_EXT.create(ext); + PackFile preserved = nonPreserved + .createPreservedForDirectory(TEST_PRESERVED_DIR); + File expected = new File(TEST_PRESERVED_DIR, + PREFIX + TEST_ID + "." + OLD_PREFIX + ext.getExtension()); + assertEquals(preserved.getName(), expected.getName()); + assertEquals(preserved.getId(), TEST_ID); + assertEquals(preserved.getPackExt(), nonPreserved.getPackExt()); + } + } + + @Test + public void canCreateAnyPreservedExtFromAnyPreservedExt() throws Exception { + // Preserved PackFiles must have an extension + PackFile preserved = new PackFile(TEST_PRESERVED_DIR, OLD_PACK); + for (PackExt from : PackExt.values()) { + PackFile preservedWithExt = preserved.create(from); + for (PackExt to : PackExt.values()) { + PackFile preservedNewExt = preservedWithExt.create(to); + File expected = new File(TEST_PRESERVED_DIR, PREFIX + TEST_ID + + "." + OLD_PREFIX + to.getExtension()); + assertEquals(preservedNewExt.getPackExt(), to); + assertEquals(preservedWithExt.getId(), preservedNewExt.getId()); + assertEquals(preservedNewExt.getName(), expected.getName()); + } + } + } + + @Test + public void canCreateNonPreservedFromAnyPreservedExt() throws Exception { + // Preserved PackFiles must have an extension + PackFile preserved = new PackFile(TEST_PRESERVED_DIR, OLD_PACK); + for (PackExt ext : PackExt.values()) { + PackFile preservedWithExt = preserved.create(ext); + PackFile nonPreserved = preservedWithExt + .createForDirectory(TEST_PACK_DIR); + File expected = new File(TEST_PACK_DIR, + PREFIX + TEST_ID + "." + ext.getExtension()); + assertEquals(nonPreserved.getName(), expected.getName()); + assertEquals(nonPreserved.getId(), TEST_ID); + assertEquals(nonPreserved.getPackExt(), + preservedWithExt.getPackExt()); + } + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index 182e42265..85bd31d71 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -246,8 +246,8 @@ public void testDelta_FailsOver2GiB() throws Exception { File dir = new File(repo.getObjectDatabase().getDirectory(), "pack"); - File packName = new File(dir, idA.name() + ".pack"); - File idxName = new File(dir, idA.name() + ".idx"); + PackFile packName = new PackFile(dir, idA.name() + ".pack"); + PackFile idxName = packName.create(PackExt.INDEX); try (FileOutputStream f = new FileOutputStream(packName)) { f.write(packContents.toByteArray()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index 214ddb989..6e8584a9c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -34,6 +34,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.TestRepository; @@ -305,9 +306,9 @@ public void testWritePack2DeltasReuseOffsets() throws IOException { @Test public void testWritePack2DeltasCRC32Copy() throws IOException { final File packDir = db.getObjectDatabase().getPackDirectory(); - final File crc32Pack = new File(packDir, + final PackFile crc32Pack = new PackFile(packDir, "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack"); - final File crc32Idx = new File(packDir, + final PackFile crc32Idx = new PackFile(packDir, "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx"); copyFile(JGitTestUtil.getTestResourceFile( "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"), @@ -471,10 +472,8 @@ public void testWriteIndex() throws Exception { config.setIndexVersion(2); writeVerifyPack4(false); - File packFile = pack.getPackFile(); - String name = packFile.getName(); - String base = name.substring(0, name.lastIndexOf('.')); - File indexFile = new File(packFile.getParentFile(), base + ".idx"); + PackFile packFile = pack.getPackFile(); + PackFile indexFile = packFile.create(PackExt.INDEX); // Validate that IndexPack came up with the right CRC32 value. final PackIndex idx1 = PackIndex.open(indexFile); @@ -687,12 +686,12 @@ private static PackIndex writePack(FileRepository repo, RevWalk walk, pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE); String id = pw.computeName().getName(); File packdir = repo.getObjectDatabase().getPackDirectory(); - File packFile = new File(packdir, "pack-" + id + ".pack"); + PackFile packFile = new PackFile(packdir, "pack-" + id + ".pack"); try (FileOutputStream packOS = new FileOutputStream(packFile)) { pw.writePack(NullProgressMonitor.INSTANCE, NullProgressMonitor.INSTANCE, packOS); } - File idxFile = new File(packdir, "pack-" + id + ".idx"); + PackFile idxFile = packFile.create(PackExt.INDEX); try (FileOutputStream idxOS = new FileOutputStream(idxFile)) { pw.writeIndex(idxOS); } diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index c00203dd0..9695e5742 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -743,6 +743,7 @@ unmergedPath=Unmerged path: {0} unmergedPaths=Repository contains unmerged paths unpackException=Exception while parsing pack stream unreadablePackIndex=Unreadable pack index: {0} +unrecognizedPackExtension=Unrecognized pack extension: {0} unrecognizedRef=Unrecognized ref: {0} unsetMark=Mark not set unsupportedAlternates=Alternates not supported diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 9d215ca45..95265feb4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -771,6 +771,7 @@ public static JGitText get() { /***/ public String unmergedPaths; /***/ public String unpackException; /***/ public String unreadablePackIndex; + /***/ public String unrecognizedPackExtension; /***/ public String unrecognizedRef; /***/ public String unsetMark; /***/ public String unsupportedAlternates; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index d928633a7..fa938b311 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -78,13 +78,11 @@ public class Pack implements Iterable { public static final Comparator SORT = (a, b) -> b.packLastModified .compareTo(a.packLastModified); - private final File packFile; + private final PackFile packFile; private final int extensions; - private File keepFile; - - private volatile String packName; + private PackFile keepFile; final int hash; @@ -137,7 +135,7 @@ public class Pack implements Iterable { * additional pack file extensions with the same base as the pack */ public Pack(File packFile, int extensions) { - this.packFile = packFile; + this.packFile = new PackFile(packFile); this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = fileSnapshot.lastModifiedInstant(); this.extensions = extensions; @@ -156,16 +154,18 @@ private PackIndex idx() throws IOException { idx = loadedIdx; if (idx == null) { if (invalid) { - throw new PackInvalidException(packFile, invalidatingCause); + throw new PackInvalidException(packFile, + invalidatingCause); } try { long start = System.currentTimeMillis(); - idx = PackIndex.open(extFile(INDEX)); + PackFile idxFile = packFile.create(INDEX); + idx = PackIndex.open(idxFile); if (LOG.isDebugEnabled()) { LOG.debug(String.format( "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ - extFile(INDEX).getAbsolutePath(), - Float.valueOf(extFile(INDEX).length() + idxFile.getAbsolutePath(), + Float.valueOf(idxFile.length() / (1024f * 1024)), Long.valueOf(System.currentTimeMillis() - start))); @@ -205,7 +205,7 @@ private PackIndex idx() throws IOException { * * @return the File object which locates this pack on disk. */ - public File getPackFile() { + public PackFile getPackFile() { return packFile; } @@ -225,16 +225,7 @@ public PackIndex getIndex() throws IOException { * @return name extracted from {@code pack-*.pack} pattern. */ public String getPackName() { - String name = packName; - if (name == null) { - name = getPackFile().getName(); - if (name.startsWith("pack-")) //$NON-NLS-1$ - name = name.substring("pack-".length()); //$NON-NLS-1$ - if (name.endsWith(".pack")) //$NON-NLS-1$ - name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$ - packName = name; - } - return name; + return packFile.getId(); } /** @@ -261,8 +252,9 @@ public boolean hasObject(AnyObjectId id) throws IOException { * @return true if a .keep file exist. */ public boolean shouldBeKept() { - if (keepFile == null) - keepFile = extFile(KEEP); + if (keepFile == null) { + keepFile = packFile.create(KEEP); + } return keepFile.exists(); } @@ -1137,7 +1129,7 @@ synchronized PackBitmapIndex getBitmapIndex() throws IOException { if (bitmapIdx == null && hasExt(BITMAP_INDEX)) { final PackBitmapIndex idx; try { - idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(), + idx = PackBitmapIndex.open(packFile.create(BITMAP_INDEX), idx(), getReverseIdx()); } catch (FileNotFoundException e) { // Once upon a time this bitmap file existed. Now it @@ -1187,13 +1179,6 @@ private void setCorrupt(long offset) { } } - private File extFile(PackExt ext) { - String p = packFile.getName(); - int dot = p.lastIndexOf('.'); - String b = (dot < 0) ? p : p.substring(0, dot); - return new File(packFile.getParentFile(), b + '.' + ext.getExtension()); - } - private boolean hasExt(PackExt ext) { return (extensions & ext.getBit()) != 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java new file mode 100644 index 000000000..c2e6f324d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import java.io.File; +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * A pack file (or pack related) File. + * + * Example: "pack-0123456789012345678901234567890123456789.idx" + */ +public class PackFile extends File { + private static final long serialVersionUID = 1L; + + private static final String PREFIX = "pack-"; //$NON-NLS-1$ + + private final String base; // PREFIX + id i.e. + // pack-0123456789012345678901234567890123456789 + + private final String id; // i.e. 0123456789012345678901234567890123456789 + + private final boolean hasOldPrefix; + + private final PackExt packExt; + + /** + * Create a PackFile for a pack or related file. + * + * @param file + * File pointing to the location of the file. + */ + public PackFile(File file) { + this(file.getParentFile(), file.getName()); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param name + * Filename (last path section) of the PackFile + */ + public PackFile(File directory, String name) { + super(directory, name); + int dot = name.lastIndexOf('.'); + + if (dot < 0) { + base = name; + hasOldPrefix = false; + packExt = null; + } else { + base = name.substring(0, dot); + String tail = name.substring(dot + 1); // ["old-"] + extension + packExt = getPackExt(tail); + String old = tail.substring(0, + tail.length() - getExtension().length()); + hasOldPrefix = old.equals(getExtPrefix(true)); + } + + id = base.startsWith(PREFIX) ? base.substring(PREFIX.length()) : base; + } + + /** + * Getter for the field id. + * + * @return the id (40 Hex char) section of the name. + */ + public String getId() { + return id; + } + + /** + * Getter for the field packExt. + * + * @return the packExt of the name. + */ + public PackExt getPackExt() { + return packExt; + } + + /** + * Create a new similar PackFile with the given extension instead. + * + * @param ext + * PackExt the extension to use. + * @return a PackFile instance with specified extension + */ + public PackFile create(PackExt ext) { + return new PackFile(getParentFile(), getName(ext)); + } + + /** + * Create a new similar PackFile in the given directory. + * + * @param directory + * Directory to create the new PackFile in. + * @return a PackFile in the given directory + */ + public PackFile createForDirectory(File directory) { + return new PackFile(directory, getName(false)); + } + + /** + * Create a new similar preserved PackFile in the given directory. + * + * @param directory + * Directory to create the new PackFile in. + * @return a PackFile in the given directory with "old-" prefixing the + * extension + */ + public PackFile createPreservedForDirectory(File directory) { + return new PackFile(directory, getName(true)); + } + + private String getName(PackExt ext) { + return base + '.' + getExtPrefix(hasOldPrefix) + ext.getExtension(); + } + + private String getName(boolean isPreserved) { + return base + '.' + getExtPrefix(isPreserved) + getExtension(); + } + + private String getExtension() { + return packExt == null ? "" : packExt.getExtension(); //$NON-NLS-1$ + } + + private static String getExtPrefix(boolean isPreserved) { + return isPreserved ? "old-" : ""; //$NON-NLS-1$ //$NON-NLS-2$ + } + + private static PackExt getPackExt(String endsWithExtension) { + for (PackExt ext : PackExt.values()) { + if (endsWithExtension.endsWith(ext.getExtension())) { + return ext; + } + } + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().unrecognizedPackExtension, endsWithExtension)); + } +} From dc7f0bfee9b3e3aed964d3fd03e765e8175de75d Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Thu, 18 Feb 2021 17:36:49 -0700 Subject: [PATCH 02/11] GC: Use PackFile to de-dup logic GC has several places where it tries to build files names for packs that we can use the PackFile class for instead. Change-Id: I99e5ceff9050f8583368fca35279251955e4644d Signed-off-by: Nasser Grainawi --- .../jgit/internal/storage/file/GC.java | 41 ++++++++----------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 75de3be89..e328870da 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -346,7 +346,7 @@ private void deleteOldPacks(Collection oldPacks, if (shouldLoosen) { loosen(inserter, reader, oldPack, ids); } - prunePack(oldName); + prunePack(oldPack.getPackFile()); } } @@ -360,19 +360,17 @@ private void deleteOldPacks(Collection oldPacks, * moves the pack file to the preserved directory * * @param packFile - * @param packName - * @param ext * @param deleteOptions * @throws IOException */ - private void removeOldPack(File packFile, String packName, PackExt ext, - int deleteOptions) throws IOException { + private void removeOldPack(PackFile packFile, int deleteOptions) + throws IOException { if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); - String oldPackName = "pack-" + packName + ".old-" + ext.getExtension(); //$NON-NLS-1$ //$NON-NLS-2$ - File oldPackFile = new File(oldPackDir, oldPackName); + PackFile oldPackFile = packFile + .createPreservedForDirectory(oldPackDir); FileUtils.rename(packFile, oldPackFile); } else { FileUtils.delete(packFile, deleteOptions); @@ -401,27 +399,21 @@ private void prunePreserved() { * ".index" file and when failing to delete the ".pack" file we are left * with a ".pack" file without a ".index" file. * - * @param packName + * @param packFile */ - private void prunePack(String packName) { - PackExt[] extensions = PackExt.values(); + private void prunePack(PackFile packFile) { try { // Delete the .pack file first and if this fails give up on deleting // the other files int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING; - for (PackExt ext : extensions) - if (PackExt.PACK.equals(ext)) { - File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - removeOldPack(f, packName, ext, deleteOptions); - break; - } + removeOldPack(packFile.create(PackExt.PACK), deleteOptions); + // The .pack file has been deleted. Delete as many as the other // files as you can. deleteOptions |= FileUtils.IGNORE_ERRORS; - for (PackExt ext : extensions) { + for (PackExt ext : PackExt.values()) { if (!PackExt.PACK.equals(ext)) { - File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - removeOldPack(f, packName, ext, deleteOptions); + removeOldPack(packFile.create(ext), deleteOptions); } } } catch (IOException e) { @@ -1218,7 +1210,7 @@ private Pack writePack(@NonNull Set want, } // rename the temporary files to real files - File realPack = nameFor(id, ".pack"); //$NON-NLS-1$ + File realPack = nameFor(id, PackExt.PACK); repo.getObjectDatabase().closeAllPackHandles(realPack); tmpPack.setReadOnly(); @@ -1228,8 +1220,7 @@ private Pack writePack(@NonNull Set want, File tmpExt = tmpEntry.getValue(); tmpExt.setReadOnly(); - File realExt = nameFor(id, - "." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$ + File realExt = nameFor(id, tmpEntry.getKey()); try { FileUtils.rename(tmpExt, realExt, StandardCopyOption.ATOMIC_MOVE); @@ -1275,9 +1266,9 @@ private Pack writePack(@NonNull Set want, } } - private File nameFor(String name, String ext) { - File packdir = repo.getObjectDatabase().getPackDirectory(); - return new File(packdir, "pack-" + name + ext); //$NON-NLS-1$ + private PackFile nameFor(String name, PackExt ext) { + return new PackFile(repo.getObjectDatabase().getPackDirectory(), + "pack-" + name).create(ext); //$NON-NLS-1$ } private void checkCancelled() throws CancelledException { From 49c89285a73401129fa57ed584abdb177961fd16 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Wed, 10 Feb 2021 23:33:43 -0700 Subject: [PATCH 03/11] PackDirectory: Use PackFile to ensure we find preserved packs Update scanPacksImpl and listPackDirectory (renamed to getPackFilesByExtById) to use the new PackFile functionality to validate file names and complete pack file sets (.pack, .idx, etc). Most importantly, this allows a later change to rely on scanPacks() to complete a packList that contains packs with the 'old-' prefix in their extension. This also eliminates duplication of logic for how to identify and construct pack files. Change-Id: I7175e5fefb187a29e0a7cf53c392aee922314f31 Signed-off-by: Nasser Grainawi --- .../internal/storage/file/PackDirectory.java | 74 +++++++++++-------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index b2ba36bf9..2e68d4624 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import java.io.File; @@ -21,7 +23,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -398,43 +399,32 @@ private PackList scanPacks(PackList original) { private PackList scanPacksImpl(PackList old) { final Map forReuse = reuseMap(old); final FileSnapshot snapshot = FileSnapshot.save(directory); - final Set names = listPackDirectory(); - final List list = new ArrayList<>(names.size() >> 2); + Map> packFilesByExtById = getPackFilesByExtById(); + List list = new ArrayList<>(packFilesByExtById.size()); boolean foundNew = false; - for (String indexName : names) { - // Must match "pack-[0-9a-f]{40}.idx" to be an index. - // - if (indexName.length() != 49 || !indexName.endsWith(".idx")) { //$NON-NLS-1$ - continue; - } - - final String base = indexName.substring(0, indexName.length() - 3); - int extensions = 0; - for (PackExt ext : PackExt.values()) { - if (names.contains(base + ext.getExtension())) { - extensions |= ext.getBit(); - } - } - - if ((extensions & PACK.getBit()) == 0) { + for (Map packFilesByExt : packFilesByExtById + .values()) { + PackFile packFile = packFilesByExt.get(PACK); + if (packFile == null || !packFilesByExt.containsKey(INDEX)) { // Sometimes C Git's HTTP fetch transport leaves a // .idx file behind and does not download the .pack. // We have to skip over such useless indexes. - // + // Also skip if we don't have any index for this id continue; } - final String packName = base + PACK.getExtension(); - final File packFile = new File(directory, packName); - final Pack oldPack = forReuse.get(packName); + Pack oldPack = forReuse.get(packFile.getName()); if (oldPack != null && !oldPack.getFileSnapshot().isModified(packFile)) { - forReuse.remove(packName); + forReuse.remove(packFile.getName()); list.add(oldPack); continue; } - list.add(new Pack(packFile, extensions)); + list.add(new Pack(packFile, + packFilesByExt.containsKey(BITMAP_INDEX) + ? BITMAP_INDEX.getBit() + : 0)); foundNew = true; } @@ -487,18 +477,40 @@ private static Map reuseMap(PackList old) { return forReuse; } - private Set listPackDirectory() { + /** + * Scans the pack directory for + * {@link org.eclipse.jgit.internal.storage.file.PackFile}s and returns them + * organized by their extensions and their pack ids + * + * Skips files in the directory that we cannot create a + * {@link org.eclipse.jgit.internal.storage.file.PackFile} for. + * + * @return a map of {@link org.eclipse.jgit.internal.storage.file.PackFile}s + * and {@link org.eclipse.jgit.internal.storage.pack.PackExt}s keyed + * by pack ids + */ + private Map> getPackFilesByExtById() { final String[] nameList = directory.list(); if (nameList == null) { - return Collections.emptySet(); + return Collections.emptyMap(); } - final Set nameSet = new HashSet<>(nameList.length << 1); + Map> packFilesByExtById = new HashMap<>( + nameList.length / 2); // assume roughly 2 files per id for (String name : nameList) { - if (name.startsWith("pack-")) { //$NON-NLS-1$ - nameSet.add(name); + try { + PackFile pack = new PackFile(directory, name); + Map packByExt = packFilesByExtById + .get(pack.getId()); + if (packByExt == null) { + packByExt = new HashMap<>(PackExt.values().length); + packFilesByExtById.put(pack.getId(), packByExt); + } + packByExt.put(pack.getPackExt(), pack); + } catch (IllegalArgumentException e) { + continue; } } - return nameSet; + return packFilesByExtById; } static final class PackList { From 7fbff35887a4179d33b17fce96191c82b397ebd7 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Wed, 10 Feb 2021 23:26:17 -0700 Subject: [PATCH 04/11] Pack: Replace extensions bitset with bitmapIdx PackFile The only extension that was ever consulted from the bitmap was the bitmap index. We can simplify the Pack code as well as the code of all the callers if we focus on just that usage. Change-Id: I799ddfdee93142af67ce5081d14a430d36aa4c15 Signed-off-by: Nasser Grainawi --- .../jgit/internal/storage/file/PackTest.java | 2 +- .../storage/file/ObjectDirectory.java | 39 +++++++++---------- .../jgit/internal/storage/file/Pack.java | 38 ++++++++---------- .../internal/storage/file/PackDirectory.java | 5 +-- 4 files changed, 38 insertions(+), 46 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index 85bd31d71..a3596541f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -261,7 +261,7 @@ public void testDelta_FailsOver2GiB() throws Exception { new PackIndexWriterV1(f).write(list, footer); } - Pack pack = new Pack(packName, PackExt.INDEX.getBit()); + Pack pack = new Pack(packName, null); try { pack.get(wc, b); fail("expected LargeObjectException.ExceedsByteArrayLimit"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index e71a96060..170df5780 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -11,8 +11,8 @@ package org.eclipse.jgit.internal.storage.file; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import java.io.BufferedReader; import java.io.File; @@ -31,7 +31,6 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; @@ -216,26 +215,26 @@ public Collection getPacks() { * Add a single existing pack to the list of available pack files. */ @Override - public Pack openPack(File pack) - throws IOException { - final String p = pack.getName(); - if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ - throw new IOException(MessageFormat.format(JGitText.get().notAValidPack, pack)); - - // The pack and index are assumed to exist. The existence of other - // extensions needs to be explicitly checked. - // - int extensions = PACK.getBit() | INDEX.getBit(); - final String base = p.substring(0, p.length() - 4); - for (PackExt ext : PackExt.values()) { - if ((extensions & ext.getBit()) == 0) { - final String name = base + ext.getExtension(); - if (new File(pack.getParentFile(), name).exists()) - extensions |= ext.getBit(); - } + public Pack openPack(File pack) throws IOException { + PackFile pf; + try { + pf = new PackFile(pack); + } catch (IllegalArgumentException e) { + throw new IOException( + MessageFormat.format(JGitText.get().notAValidPack, pack), + e); } - Pack res = new Pack(pack, extensions); + String p = pf.getName(); + // TODO(nasserg): See if PackFile can do these checks instead + if (p.length() != 50 || !p.startsWith("pack-") //$NON-NLS-1$ + || !pf.getPackExt().equals(PACK)) { + throw new IOException( + MessageFormat.format(JGitText.get().notAValidPack, pack)); + } + + PackFile bitmapIdx = pf.create(BITMAP_INDEX); + Pack res = new Pack(pack, bitmapIdx.exists() ? bitmapIdx : null); packed.insert(res); return res; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index fa938b311..5efd4c5bf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; @@ -38,6 +37,7 @@ import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; @@ -51,7 +51,6 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.BinaryDelta; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; @@ -80,8 +79,6 @@ public class Pack implements Iterable { private final PackFile packFile; - private final int extensions; - private PackFile keepFile; final int hash; @@ -105,7 +102,8 @@ public class Pack implements Iterable { private volatile Exception invalidatingCause; - private boolean invalidBitmap; + @Nullable + private PackFile bitmapIdxFile; private AtomicInteger transientErrorCount = new AtomicInteger(); @@ -131,14 +129,14 @@ public class Pack implements Iterable { * * @param packFile * path of the .pack file holding the data. - * @param extensions - * additional pack file extensions with the same base as the pack + * @param bitmapIdxFile + * existing bitmap index file with the same base as the pack */ - public Pack(File packFile, int extensions) { + public Pack(File packFile, @Nullable PackFile bitmapIdxFile) { this.packFile = new PackFile(packFile); this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = fileSnapshot.lastModifiedInstant(); - this.extensions = extensions; + this.bitmapIdxFile = bitmapIdxFile; // Multiply by 31 here so we can more directly combine with another // value in WindowCache.hash(), without doing the multiply there. @@ -1124,26 +1122,28 @@ private long findEndOffset(long startOffset) } synchronized PackBitmapIndex getBitmapIndex() throws IOException { - if (invalid || invalidBitmap) + if (invalid || bitmapIdxFile == null) { return null; - if (bitmapIdx == null && hasExt(BITMAP_INDEX)) { + } + if (bitmapIdx == null) { final PackBitmapIndex idx; try { - idx = PackBitmapIndex.open(packFile.create(BITMAP_INDEX), idx(), + idx = PackBitmapIndex.open(bitmapIdxFile, idx(), getReverseIdx()); } catch (FileNotFoundException e) { // Once upon a time this bitmap file existed. Now it // has been removed. Most likely an external gc has // removed this packfile and the bitmap - invalidBitmap = true; - return null; + bitmapIdxFile = null; + return null; } // At this point, idx() will have set packChecksum. - if (Arrays.equals(packChecksum, idx.packChecksum)) + if (Arrays.equals(packChecksum, idx.packChecksum)) { bitmapIdx = idx; - else - invalidBitmap = true; + } else { + bitmapIdxFile = null; + } } return bitmapIdx; } @@ -1179,10 +1179,6 @@ private void setCorrupt(long offset) { } } - private boolean hasExt(PackExt ext) { - return (extensions & ext.getBit()) != 0; - } - @SuppressWarnings("nls") @Override public String toString() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 2e68d4624..007205e55 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -421,10 +421,7 @@ private PackList scanPacksImpl(PackList old) { continue; } - list.add(new Pack(packFile, - packFilesByExt.containsKey(BITMAP_INDEX) - ? BITMAP_INDEX.getBit() - : 0)); + list.add(new Pack(packFile, packFilesByExt.get(BITMAP_INDEX))); foundNew = true; } From 6167641834e28f8ad322f8fde60866b339bfb7fe Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Tue, 15 Dec 2020 14:20:44 -0700 Subject: [PATCH 05/11] Restore preserved packs during missing object seeks Provide a recovery path for objects being referenced during the pack pruning race. Due to the pack pruning race, it is possible for objects to become referenced after a pack has been deemed safe to prune, but before it actually gets pruned. If this happened previously, the newly referenced objects would be missing and potentially result in a corrupted ref. Add the ability to recover from this situation when an object is missing but happens to still be available in a pack in the "preserved" directory. This is likely only useful when used in conjunction with the --preserve-old-packs GC option, which prunes packs by hard-linking to the preserved directory. If an object is missing and found in a pack in the preserved directory, immediately recover that pack and its associated files (idx, bitmaps...) by moving them back to the original pack directory, and then retry the operation that would have failed due to the missing object. This retry can now succeed and the repository may avoid corruption. This approach should drastically reduce the chance of a corrupt repository during pack pruning at very little extra cost. This extra cost should only be incurred when objects are missing and a failure would normally occur. Change-Id: I2a704e3276b88cc892159d9bfe2455c6eec64252 Signed-off-by: Martin Fick Signed-off-by: Nasser Grainawi --- .../storage/file/GcBasicPackingTest.java | 51 ++++++++++++ .../storage/file/ObjectDirectory.java | 83 +++++++++++++++++-- .../internal/storage/file/PackDirectory.java | 28 +++++-- 3 files changed, 150 insertions(+), 12 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index 61538c9c2..8dc1ddb9f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -23,6 +23,7 @@ import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -314,6 +315,56 @@ private void testPreserveOldPacks() throws Exception { assertTrue(preservedPackFile.exists()); } + @Test + public void testPruneAndRestoreOldPacks() throws Exception { + String tempRef = "refs/heads/soon-to-be-unreferenced"; + BranchBuilder bb = tr.branch(tempRef); + bb.commit().add("A", "A").add("B", "B").create(); + + // Verify setup conditions + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + + // Force all referenced objects into packs (to avoid having loose objects) + configureGc(gc, false); + gc.setExpireAgeMillis(0); + gc.setPackExpireAgeMillis(0); + gc.gc(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + + // Delete the temp ref, orphaning its commit + RefUpdate update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setForceUpdate(true); + ObjectId objectId = update.getOldObjectId(); // remember it so we can restore it! + RefUpdate.Result result = update.delete(); + assertEquals(RefUpdate.Result.FORCED, result); + + fsTick(); + + // Repack with only orphaned commit, so packfile will be pruned + configureGc(gc, false).setPreserveOldPacks(true); + gc.gc(); + stats = gc.getStatistics(); + assertEquals(0, stats.numberOfLooseObjects); + assertEquals(0, stats.numberOfPackedObjects); + assertEquals(0, stats.numberOfPackFiles); + + // Restore the temp ref to the deleted commit, should restore old-packs! + update = tr.getRepository().getRefDatabase().newUpdate(tempRef, false); + update.setNewObjectId(objectId); + update.setExpectedOldObjectId(null); + result = update.update(); + assertEquals(RefUpdate.Result.NEW, result); + + stats = gc.getStatistics(); + assertEquals(4, stats.numberOfPackedObjects); + assertEquals(1, stats.numberOfPackFiles); + } + private PackConfig configureGc(GC myGc, boolean aggressive) { PackConfig pconfig = new PackConfig(repo); if (aggressive) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 170df5780..627facca0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -13,6 +13,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.BufferedReader; import java.io.File; @@ -31,6 +32,7 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; @@ -78,7 +80,7 @@ public class ObjectDirectory extends FileObjectDatabase { private final PackDirectory packed; - private final File preservedDirectory; + private final PackDirectory preserved; private final File alternatesFile; @@ -116,10 +118,11 @@ public ObjectDirectory(final Config cfg, final File dir, objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ - preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ + File preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); loose = new LooseObjects(objects); packed = new PackDirectory(config, packDirectory); + preserved = new PackDirectory(config, preservedDirectory); this.fs = fs; this.shallowFile = shallowFile; @@ -155,7 +158,7 @@ public final File getPackDirectory() { * @return the location of the preserved directory. */ public final File getPreservedDirectory() { - return preservedDirectory; + return preserved.getDirectory(); } /** {@inheritDoc} */ @@ -249,7 +252,13 @@ public String toString() { @Override public boolean has(AnyObjectId objectId) { return loose.hasCached(objectId) - || hasPackedInSelfOrAlternate(objectId, null) + || hasPackedOrLooseInSelfOrAlternate(objectId) + || (restoreFromSelfOrAlternate(objectId, null) + && hasPackedOrLooseInSelfOrAlternate(objectId)); + } + + private boolean hasPackedOrLooseInSelfOrAlternate(AnyObjectId objectId) { + return hasPackedInSelfOrAlternate(objectId, null) || hasLooseInSelfOrAlternate(objectId, null); } @@ -318,6 +327,15 @@ private void resolve(Set matches, AbbreviatedObjectId id, @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { + ObjectLoader ldr = openObjectWithoutRestoring(curs, objectId); + if (ldr == null && restoreFromSelfOrAlternate(objectId, null)) { + ldr = openObjectWithoutRestoring(curs, objectId); + } + return ldr; + } + + private ObjectLoader openObjectWithoutRestoring(WindowCursor curs, AnyObjectId objectId) + throws IOException { if (loose.hasCached(objectId)) { ObjectLoader ldr = openLooseObject(curs, objectId); if (ldr != null) { @@ -379,8 +397,16 @@ ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id) } @Override - long getObjectSize(WindowCursor curs, AnyObjectId id) - throws IOException { + long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { + long sz = getObjectSizeWithoutRestoring(curs, id); + if (0 > sz && restoreFromSelfOrAlternate(id, null)) { + sz = getObjectSizeWithoutRestoring(curs, id); + } + return sz; + } + + private long getObjectSizeWithoutRestoring(WindowCursor curs, + AnyObjectId id) throws IOException { if (loose.hasCached(id)) { long len = loose.getSize(curs, id); if (0 <= len) { @@ -448,6 +474,51 @@ private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, } } + private boolean restoreFromSelfOrAlternate(AnyObjectId objectId, + Set skips) { + if (restoreFromSelf(objectId)) { + return true; + } + + skips = addMe(skips); + for (AlternateHandle alt : myAlternates()) { + if (!skips.contains(alt.getId())) { + if (alt.db.restoreFromSelfOrAlternate(objectId, skips)) { + return true; + } + } + } + return false; + } + + private boolean restoreFromSelf(AnyObjectId objectId) { + Pack preservedPack = preserved.getPack(objectId); + if (preservedPack == null) { + return false; + } + PackFile preservedFile = new PackFile(preservedPack.getPackFile()); + // Restore the index last since the set will be considered for use once + // the index appears. + for (PackExt ext : PackExt.values()) { + if (!INDEX.equals(ext)) { + restore(preservedFile.create(ext)); + } + } + restore(preservedFile.create(INDEX)); + return true; + } + + private boolean restore(PackFile preservedPack) { + PackFile restored = preservedPack + .createForDirectory(packed.getDirectory()); + try { + Files.createLink(restored.toPath(), preservedPack.toPath()); + } catch (IOException e) { + return false; + } + return true; + } + @Override InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 007205e55..5f3097023 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.PackMismatchException; @@ -122,21 +123,36 @@ public String toString() { * * @param objectId * identity of the object to test for existence of. - * @return true if the specified object is stored in this PackDirectory. + * @return {@code true} if the specified object is stored in this PackDirectory. */ boolean has(AnyObjectId objectId) { + return getPack(objectId) != null; + } + + /** + * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} for the + * specified object if it is stored in this PackDirectory. + * + * @param objectId + * identity of the object to find the Pack for. + * @return {@link org.eclipse.jgit.internal.storage.file.Pack} which + * contains the specified object or {@code null} if it is not stored + * in this PackDirectory. + */ + @Nullable + Pack getPack(AnyObjectId objectId) { PackList pList; do { pList = packList.get(); for (Pack p : pList.packs) { try { if (p.hasObject(objectId)) { - return true; + return p; } } catch (IOException e) { - // The hasObject call should have only touched the index, - // so any failure here indicates the index is unreadable - // by this process, and the pack is likewise not readable. + // The hasObject call should have only touched the index, so + // any failure here indicates the index is unreadable by + // this process, and the pack is likewise not readable. LOG.warn(MessageFormat.format( JGitText.get().unableToReadPackfile, p.getPackFile().getAbsolutePath()), e); @@ -144,7 +160,7 @@ boolean has(AnyObjectId objectId) { } } } while (searchPacksAgain(pList)); - return false; + return null; } /** From c57b2935cd8bf160cb81ef86205ea0c5ea36da98 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Fri, 26 Feb 2021 15:49:06 -0700 Subject: [PATCH 06/11] PackExt: Convert to Enum This class already looked very much like an Enum, but wasn't one. As an Enum, we can use PackExt in EnumMaps and EnumSets. Convert the Map key usage in PackDirectory to an EnumMap. Change-Id: Ice097fd468a05805f914e6862fbd1d96ec8c45d1 Signed-off-by: Nasser Grainawi --- .../internal/storage/file/PackDirectory.java | 15 +++-- .../jgit/internal/storage/pack/PackExt.java | 60 ++++--------------- 2 files changed, 19 insertions(+), 56 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 5f3097023..73745d8c6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -512,13 +513,15 @@ private Map> getPackFilesByExtById() { for (String name : nameList) { try { PackFile pack = new PackFile(directory, name); - Map packByExt = packFilesByExtById - .get(pack.getId()); - if (packByExt == null) { - packByExt = new HashMap<>(PackExt.values().length); - packFilesByExtById.put(pack.getId(), packByExt); + if (pack.getPackExt() != null) { + Map packByExt = packFilesByExtById + .get(pack.getId()); + if (packByExt == null) { + packByExt = new EnumMap<>(PackExt.class); + packFilesByExtById.put(pack.getId(), packByExt); + } + packByExt.put(pack.getPackExt(), pack); } - packByExt.put(pack.getPackExt(), pack); } catch (IllegalArgumentException e) { continue; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java index bedc6939c..6fb775da8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java @@ -13,66 +13,26 @@ /** * A pack file extension. */ -public class PackExt { - private static volatile PackExt[] VALUES = new PackExt[] {}; - +public enum PackExt { /** A pack file extension. */ - public static final PackExt PACK = newPackExt("pack"); //$NON-NLS-1$ + PACK("pack"), //$NON-NLS-1$ /** A pack index file extension. */ - public static final PackExt INDEX = newPackExt("idx"); //$NON-NLS-1$ + INDEX("idx"), //$NON-NLS-1$ /** A keep pack file extension. */ - public static final PackExt KEEP = newPackExt("keep"); //$NON-NLS-1$ + KEEP("keep"), //$NON-NLS-1$ /** A pack bitmap index file extension. */ - public static final PackExt BITMAP_INDEX = newPackExt("bitmap"); //$NON-NLS-1$ + BITMAP_INDEX("bitmap"), //$NON-NLS-1$ /** A reftable file. */ - public static final PackExt REFTABLE = newPackExt("ref"); //$NON-NLS-1$ - - /** - * Get all of the PackExt values. - * - * @return all of the PackExt values. - */ - public static PackExt[] values() { - return VALUES; - } - - /** - * Returns a PackExt for the file extension and registers it in the values - * array. - * - * @param ext - * the file extension. - * @return the PackExt for the ext - */ - public static synchronized PackExt newPackExt(String ext) { - PackExt[] dst = new PackExt[VALUES.length + 1]; - for (int i = 0; i < VALUES.length; i++) { - PackExt packExt = VALUES[i]; - if (packExt.getExtension().equals(ext)) - return packExt; - dst[i] = packExt; - } - if (VALUES.length >= 32) - throw new IllegalStateException( - "maximum number of pack extensions exceeded"); //$NON-NLS-1$ - - PackExt value = new PackExt(ext, VALUES.length); - dst[VALUES.length] = value; - VALUES = dst; - return value; - } + REFTABLE("ref"); //$NON-NLS-1$ private final String ext; - private final int pos; - - private PackExt(String ext, int pos) { + private PackExt(String ext) { this.ext = ext; - this.pos = pos; } /** @@ -85,12 +45,12 @@ public String getExtension() { } /** - * Get the position of the extension in the values array. + * Get the position of the extension in the enum declaration. * - * @return the position of the extension in the values array. + * @return the position of the extension in the enum declaration. */ public int getPosition() { - return pos; + return ordinal(); } /** From 093020864f06b75708f8ec225e5df9e0ad54f9c1 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Wed, 3 Mar 2021 17:05:21 -0700 Subject: [PATCH 07/11] GC: deleteOrphans: Use PackFile It's easier to follow the logic here when we can use our own objects instead of Strings. Change-Id: I6a166edcc67903fc1ca3544f458634c4cef8fde7 Signed-off-by: Nasser Grainawi --- .../jgit/internal/storage/file/GC.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index e328870da..9366404ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -12,6 +12,8 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; import java.io.File; import java.io.FileOutputStream; @@ -965,20 +967,21 @@ private void deleteOrphans() { return; } - String base = null; + String latestId = null; for (String n : fileNames) { - if (n.endsWith(PACK_EXT) || n.endsWith(KEEP_EXT)) { - base = n.substring(0, n.lastIndexOf('.')); - } else { - if (base == null || !n.startsWith(base)) { - try { - Path delete = packDir.resolve(n); - FileUtils.delete(delete.toFile(), - FileUtils.RETRY | FileUtils.SKIP_MISSING); - LOG.warn(JGitText.get().deletedOrphanInPackDir, delete); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } + PackFile pf = new PackFile(packDir.toFile(), n); + PackExt ext = pf.getPackExt(); + if (ext.equals(PACK) || ext.equals(KEEP)) { + latestId = pf.getId(); + } + if (latestId == null || !pf.getId().equals(latestId)) { + // no pack or keep for this id + try { + FileUtils.delete(pf, + FileUtils.RETRY | FileUtils.SKIP_MISSING); + LOG.warn(JGitText.get().deletedOrphanInPackDir, pf); + } catch (IOException e) { + LOG.error(e.getMessage(), e); } } } From 2a6b2eddcfac17ef5ff3b6b28dfaadd83e34956a Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Thu, 4 Mar 2021 14:14:43 -0700 Subject: [PATCH 08/11] PackFile: Add id + ext based constructors Add new constructors to PackFile to improve a common use case where callers know the directory, id, and extension, but previously needed to construct a valid file name (with prefix, '.', etc) to create a PackFile. Most callers can use the variant that has id as an ObjectId, but provide an id as String variant too. Change-Id: I39e4466abe8c9509f5916d5bfe675066570b8585 Signed-off-by: Nasser Grainawi --- .../eclipse/jgit/junit/TestRepository.java | 11 +----- .../storage/file/AbbreviationTest.java | 6 +-- .../storage/file/ConcurrentRepackTest.java | 12 ++---- .../internal/storage/file/PackFileTest.java | 31 ++++++++++++--- .../internal/storage/file/PackWriterTest.java | 4 +- .../jgit/internal/storage/file/GC.java | 12 ++---- .../storage/file/LocalCachedPack.java | 3 +- .../file/ObjectDirectoryPackParser.java | 9 +++-- .../jgit/internal/storage/file/PackFile.java | 33 ++++++++++++++++ .../internal/storage/file/PackInserter.java | 9 +++-- .../jgit/transport/WalkPushConnection.java | 38 ++++++++++--------- 11 files changed, 106 insertions(+), 62 deletions(-) diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index 24f7741f1..0232156a4 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java @@ -915,9 +915,8 @@ public void packAndPrune() throws Exception { all.add(r.getObjectId()); pw.preparePack(m, all, PackWriter.NONE); - final ObjectId name = pw.computeName(); - - pack = nameFor(odb, name, ".pack"); + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(pack))) { pw.writePack(m, m, out); @@ -962,12 +961,6 @@ private static void prunePacked(ObjectDirectory odb) throws IOException { } } - private static PackFile nameFor(ObjectDirectory odb, ObjectId name, - String t) { - File packdir = odb.getPackDirectory(); - return new PackFile(packdir, "pack-" + name.name() + t); - } - private void writeFile(File p, byte[] bin) throws IOException, ObjectWritingException { final LockFile lck = new LockFile(p); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java index 45d864d45..bd36337f3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java @@ -28,6 +28,7 @@ import java.util.List; import org.eclipse.jgit.errors.AmbiguousObjectException; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.AbbreviatedObjectId; @@ -144,10 +145,9 @@ public void testAbbreviateIsActuallyUnique() throws Exception { objects.add(new PackedObjectInfo(ObjectId.fromRaw(idBuf))); } - String packName = "pack-" + id.name(); File packDir = db.getObjectDatabase().getPackDirectory(); - File idxFile = new File(packDir, packName + ".idx"); - File packFile = new File(packDir, packName + ".pack"); + PackFile idxFile = new PackFile(packDir, id, PackExt.INDEX); + PackFile packFile = idxFile.create(PackExt.PACK); FileUtils.mkdir(packDir, true); try (OutputStream dst = new BufferedOutputStream( new FileOutputStream(idxFile))) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java index ca6350735..df5d952ee 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java @@ -194,9 +194,10 @@ private File[] pack(Repository src, RevObject... list) pw.addObject(o); } - final ObjectId name = pw.computeName(); - final PackFile packFile = fullPackFileName(name, ".pack"); - final PackFile idxFile = packFile.create(PackExt.INDEX); + PackFile packFile = new PackFile( + db.getObjectDatabase().getPackDirectory(), pw.computeName(), + PackExt.PACK); + PackFile idxFile = packFile.create(PackExt.INDEX); final File[] files = new File[] { packFile, idxFile }; write(files, pw); return files; @@ -243,11 +244,6 @@ private static void touch(Instant begin, File dir) throws IOException { } } - private PackFile fullPackFileName(ObjectId name, String suffix) { - final File packdir = db.getObjectDatabase().getPackDirectory(); - return new PackFile(packdir, "pack-" + name.name() + suffix); - } - private RevObject writeBlob(Repository repo, String data) throws IOException { final byte[] bytes = Constants.encode(data); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java index 88d25b73d..619cfcac3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java @@ -17,10 +17,14 @@ import java.io.File; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.ObjectId; import org.junit.Test; public class PackFileTest { - private static final String TEST_ID = "0123456789012345678901234567890123456789"; + private static final ObjectId TEST_OID = ObjectId + .fromString("0123456789012345678901234567890123456789"); + + private static final String TEST_ID = TEST_OID.name(); private static final String PREFIX = "pack-"; @@ -39,12 +43,20 @@ public class PackFileTest { new File(TEST_PACK_DIR, PREFIX + TEST_ID)); @Test - public void idIsSameFromFileOrDirAndName() throws Exception { - File pack = new File(TEST_PACK_DIR, PREFIX + TEST_ID); + public void objectsAreSameFromAnyConstructor() throws Exception { + String name = PREFIX + TEST_ID + "." + PackExt.PACK.getExtension(); + File pack = new File(TEST_PACK_DIR, name); PackFile pf = new PackFile(pack); - PackFile pfFromDirAndName = new PackFile(TEST_PACK_DIR, - PREFIX + TEST_ID); - assertEquals(pf.getId(), pfFromDirAndName.getId()); + PackFile pfFromDirAndName = new PackFile(TEST_PACK_DIR, name); + assertPackFilesEqual(pf, pfFromDirAndName); + + PackFile pfFromOIdAndExt = new PackFile(TEST_PACK_DIR, TEST_OID, + PackExt.PACK); + assertPackFilesEqual(pf, pfFromOIdAndExt); + + PackFile pfFromIdAndExt = new PackFile(TEST_PACK_DIR, TEST_ID, + PackExt.PACK); + assertPackFilesEqual(pf, pfFromIdAndExt); } @Test @@ -147,4 +159,11 @@ public void canCreateNonPreservedFromAnyPreservedExt() throws Exception { preservedWithExt.getPackExt()); } } + + private void assertPackFilesEqual(PackFile p1, PackFile p2) { + // for test purposes, considered equal if id, name, and ext are equal + assertEquals(p1.getId(), p2.getId()); + assertEquals(p1.getPackExt(), p2.getPackExt()); + assertEquals(p1.getName(), p2.getName()); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index 6e8584a9c..e422ab9db 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -684,9 +684,9 @@ private static PackIndex writePack(FileRepository repo, RevWalk walk, ObjectWalk ow = walk.toObjectWalkWithSameObjects(); pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE); - String id = pw.computeName().getName(); File packdir = repo.getObjectDatabase().getPackDirectory(); - PackFile packFile = new PackFile(packdir, "pack-" + id + ".pack"); + PackFile packFile = new PackFile(packdir, pw.computeName(), + PackExt.PACK); try (FileOutputStream packOS = new FileOutputStream(packFile)) { pw.writePack(NullProgressMonitor.INSTANCE, NullProgressMonitor.INSTANCE, packOS); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 9366404ba..9ffff9f66 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1163,7 +1163,7 @@ private Pack writePack(@NonNull Set want, checkCancelled(); // create temporary files - String id = pw.computeName().getName(); + ObjectId id = pw.computeName(); File packdir = repo.getObjectDatabase().getPackDirectory(); packdir.mkdirs(); tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$ @@ -1213,7 +1213,8 @@ private Pack writePack(@NonNull Set want, } // rename the temporary files to real files - File realPack = nameFor(id, PackExt.PACK); + File packDir = repo.getObjectDatabase().getPackDirectory(); + PackFile realPack = new PackFile(packDir, id, PackExt.PACK); repo.getObjectDatabase().closeAllPackHandles(realPack); tmpPack.setReadOnly(); @@ -1223,7 +1224,7 @@ private Pack writePack(@NonNull Set want, File tmpExt = tmpEntry.getValue(); tmpExt.setReadOnly(); - File realExt = nameFor(id, tmpEntry.getKey()); + PackFile realExt = new PackFile(packDir, id, tmpEntry.getKey()); try { FileUtils.rename(tmpExt, realExt, StandardCopyOption.ATOMIC_MOVE); @@ -1269,11 +1270,6 @@ private Pack writePack(@NonNull Set want, } } - private PackFile nameFor(String name, PackExt ext) { - return new PackFile(repo.getObjectDatabase().getPackDirectory(), - "pack-" + name).create(ext); //$NON-NLS-1$ - } - private void checkCancelled() throws CancelledException { if (pm.isCancelled() || Thread.currentThread().isInterrupted()) { throw new CancelledException(JGitText.get().operationCanceled); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index ae5bce698..f112947ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -17,6 +17,7 @@ import org.eclipse.jgit.internal.storage.pack.CachedPack; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation; @@ -88,6 +89,6 @@ private Pack getPackFile(String packName) throws FileNotFoundException { private String getPackFilePath(String packName) { final File packDir = odb.getPackDirectory(); - return new File(packDir, "pack-" + packName + ".pack").getPath(); //$NON-NLS-1$ //$NON-NLS-2$ + return new PackFile(packDir, packName, PackExt.PACK).getPath(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 04d2ff8ab..dba8ccd99 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -27,6 +27,7 @@ import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -426,10 +427,10 @@ private PackLock renameAndOpenPack(String lockMessage) d.update(oeBytes); } - final String name = ObjectId.fromRaw(d.digest()).name(); - final File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$ - final File finalPack = new File(packDir, "pack-" + name + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ - final File finalIdx = new File(packDir, "pack-" + name + ".idx"); //$NON-NLS-1$ //$NON-NLS-2$ + ObjectId id = ObjectId.fromRaw(d.digest()); + File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$ + PackFile finalPack = new PackFile(packDir, id, PackExt.PACK); + PackFile finalIdx = finalPack.create(PackExt.INDEX); final PackLock keep = new PackLock(finalPack, db.getFS()); if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index c2e6f324d..19979d0ed 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -16,6 +16,7 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.ObjectId; /** * A pack file (or pack related) File. @@ -36,6 +37,10 @@ public class PackFile extends File { private final PackExt packExt; + private static String createName(String id, PackExt extension) { + return PREFIX + id + '.' + extension.getExtension(); + } + /** * Create a PackFile for a pack or related file. * @@ -46,6 +51,34 @@ public PackFile(File file) { this(file.getParentFile(), file.getName()); } + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param id + * the {@link ObjectId} for this pack + * @param ext + * the packExt of the name. + */ + public PackFile(File directory, ObjectId id, PackExt ext) { + this(directory, id.name(), ext); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param id + * the id (40 Hex char) section of the pack name. + * @param ext + * the packExt of the name. + */ + public PackFile(File directory, String id, PackExt ext) { + this(directory, createName(id, ext)); + } + /** * Create a PackFile for a pack or related file. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index a27a2b00c..d6209c4a7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -76,6 +76,7 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -273,16 +274,16 @@ public void flush() throws IOException { } Collections.sort(objectList); - File tmpIdx = idxFor(tmpPack); + File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - File realPack = new File(db.getPackDirectory(), - "pack-" + computeName(objectList).name() + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ + PackFile realPack = new PackFile(db.getPackDirectory(), + computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); tmpPack.setReadOnly(); FileUtils.rename(tmpPack, realPack, ATOMIC_MOVE); - File realIdx = idxFor(realPack); + PackFile realIdx = realPack.create(PackExt.INDEX); tmpIdx.setReadOnly(); try { FileUtils.rename(tmpIdx, realIdx, ATOMIC_MOVE); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java index f2eac8d24..03ef852c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java @@ -13,6 +13,7 @@ import static org.eclipse.jgit.transport.WalkRemoteObjectDatabase.ROOT_DIR; import java.io.BufferedOutputStream; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; @@ -26,6 +27,8 @@ import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -189,9 +192,8 @@ public void close() { private void sendpack(final List updates, final ProgressMonitor monitor) throws TransportException { - String pathPack = null; - String pathIdx = null; - + PackFile pack = null; + PackFile idx = null; try (PackWriter writer = new PackWriter(transport.getPackConfig(), local.newObjectReader())) { @@ -217,31 +219,33 @@ private void sendpack(final List updates, for (String n : dest.getPackNames()) packNames.put(n, n); - final String base = "pack-" + writer.computeName().name(); //$NON-NLS-1$ - final String packName = base + ".pack"; //$NON-NLS-1$ - pathPack = "pack/" + packName; //$NON-NLS-1$ - pathIdx = "pack/" + base + ".idx"; //$NON-NLS-1$ //$NON-NLS-2$ + File packDir = new File("pack"); //$NON-NLS-1$ + pack = new PackFile(packDir, writer.computeName(), + PackExt.PACK); + idx = pack.create(PackExt.INDEX); - if (packNames.remove(packName) != null) { + if (packNames.remove(pack.getName()) != null) { // The remote already contains this pack. We should // remove the index before overwriting to prevent bad // offsets from appearing to clients. // dest.writeInfoPacks(packNames.keySet()); - dest.deleteFile(pathIdx); + dest.deleteFile(idx.getPath()); } // Write the pack file, then the index, as readers look the // other direction (index, then pack file). // - String wt = "Put " + base.substring(0, 12); //$NON-NLS-1$ + String wt = "Put " + pack.getName().substring(0, 12); //$NON-NLS-1$ try (OutputStream os = new BufferedOutputStream( - dest.writeFile(pathPack, monitor, wt + "..pack"))) { //$NON-NLS-1$ + dest.writeFile(pack.getPath(), monitor, + wt + "." + pack.getPackExt().getExtension()))) { //$NON-NLS-1$ writer.writePack(monitor, monitor, os); } try (OutputStream os = new BufferedOutputStream( - dest.writeFile(pathIdx, monitor, wt + "..idx"))) { //$NON-NLS-1$ + dest.writeFile(idx.getPath(), monitor, + wt + "." + idx.getPackExt().getExtension()))) { //$NON-NLS-1$ writer.writeIndex(os); } @@ -250,22 +254,22 @@ private void sendpack(final List updates, // and discover the most recent objects there. // final ArrayList infoPacks = new ArrayList<>(); - infoPacks.add(packName); + infoPacks.add(pack.getName()); infoPacks.addAll(packNames.keySet()); dest.writeInfoPacks(infoPacks); } catch (IOException err) { - safeDelete(pathIdx); - safeDelete(pathPack); + safeDelete(idx); + safeDelete(pack); throw new TransportException(uri, JGitText.get().cannotStoreObjects, err); } } - private void safeDelete(String path) { + private void safeDelete(File path) { if (path != null) { try { - dest.deleteFile(path); + dest.deleteFile(path.getPath()); } catch (IOException cleanupFailure) { // Ignore the deletion failure. We probably are // already failing and were just trying to pick From 663f29e1588f2c024b46b3084ec7c5d30d0515da Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 28 Jan 2021 00:25:06 +0100 Subject: [PATCH 09/11] Add dependency to dash-licenses This is required to run the license check using the license tool [1] required by the Eclipse project handbook [2]: mvn org.eclipse.dash:license-tool-plugin:license-check -Ddash.summary=DEPENDENCIES Note: the tool still requires Java 11 hence it needs to be run in a separate build step and is not yet integrated in the build which runs on Java 8. [1] https://github.com/eclipse/dash-licenses [2] https://www.eclipse.org/projects/handbook/#ip-license-tool Change-Id: Ib41d54de246c3c9499cc3be9f026294c39fdfd99 --- pom.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pom.xml b/pom.xml index 4846aff54..d938d1a14 100644 --- a/pom.xml +++ b/pom.xml @@ -202,6 +202,14 @@ repo.eclipse.org.cbi-snapshots https://repo.eclipse.org/content/repositories/cbi-snapshots/ + + repo.eclipse.org.dash-releases + https://repo.eclipse.org/content/repositories/dash-licenses-releases/ + + + repo.eclipse.org.dash-snapshots + https://repo.eclipse.org/content/repositories/dash-licenses-snapshots/ + @@ -391,6 +399,11 @@ spring-boot-maven-plugin 2.4.1 + + org.eclipse.dash + license-tool-plugin + 0.0.1-SNAPSHOT + @@ -549,6 +562,10 @@ org.apache.maven.plugins maven-surefire-report-plugin + + org.eclipse.dash + license-tool-plugin + From 8d712c84607288ddbfa16f2df05fa58e077aa4e6 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 7 Mar 2021 17:44:05 +0100 Subject: [PATCH 10/11] Update DEPENDENCIES report for 5.11.0 Computed by dash license-tool-plugin [1]. [1] https://github.com/eclipse/dash-licenses Change-Id: I28727115914613cefdcf9a9f462c195b6af23156 --- DEPENDENCIES | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 DEPENDENCIES diff --git a/DEPENDENCIES b/DEPENDENCIES new file mode 100644 index 000000000..541a999de --- /dev/null +++ b/DEPENDENCIES @@ -0,0 +1,66 @@ +maven/mavencentral/args4j/args4j/2.33, MIT, approved, CQ11068 +maven/mavencentral/com.google.code.gson/gson/2.8.6, Apache-2.0, approved, CQ23102 +maven/mavencentral/com.googlecode.javaewah/JavaEWAH/1.1.7, Apache-2.0, approved, CQ11658 +maven/mavencentral/com.jcraft/jsch/0.1.55, BSD-3-Clause, approved, CQ19435 +maven/mavencentral/com.jcraft/jzlib/1.1.1, BSD-2-Clause, approved, CQ6218 +maven/mavencentral/commons-codec/commons-codec/1.11, Apache-2.0, approved, CQ15971 +maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162 +maven/mavencentral/javax.servlet/javax.servlet-api/3.1.0, Apache-2.0 AND (CDDL-1.1 OR GPL-2.0 WITH Classpath-exception-2.0), approved, emo_ip_team +maven/mavencentral/junit/junit/4.13, , approved, CQ22796 +maven/mavencentral/log4j/log4j/1.2.15, Apache-2.0, approved, CQ7837 +maven/mavencentral/net.bytebuddy/byte-buddy-agent/1.9.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/net.bytebuddy/byte-buddy/1.9.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/net.i2p.crypto/eddsa/0.3.0, CC0, approved, CQ17804 +maven/mavencentral/net.sf.jopt-simple/jopt-simple/4.6, MIT, approved, clearlydefined +maven/mavencentral/org.apache.ant/ant-launcher/1.10.8, Apache-2.0 AND W3C AND LicenseRef-Public-Domain, approved, CQ15560 +maven/mavencentral/org.apache.ant/ant/1.10.8, Apache-2.0 AND W3C AND LicenseRef-Public-Domain, approved, CQ15560 +maven/mavencentral/org.apache.commons/commons-compress/1.19, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.commons/commons-math3/3.2, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.httpcomponents/httpclient/4.5.13, Apache-2.0, approved, CQ22761 +maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.14, Apache-2.0, approved, CQ18704 +maven/mavencentral/org.apache.sshd/sshd-common/2.6.0, Apache-2.0 AND ISC, approved, CQ22992 +maven/mavencentral/org.apache.sshd/sshd-core/2.6.0, Apache-2.0 AND ISC, approved, CQ22992 +maven/mavencentral/org.apache.sshd/sshd-osgi/2.6.0, Apache-2.0 AND ISC, approved, CQ22992 +maven/mavencentral/org.apache.sshd/sshd-sftp/2.6.0, Apache-2.0 AND ISC, approved, CQ22993 +maven/mavencentral/org.assertj/assertj-core/3.14.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.bouncycastle/bcpg-jdk15on/1.65, Apache-2.0, approved, CQ21975 +maven/mavencentral/org.bouncycastle/bcpkix-jdk15on/1.65, MIT AND LicenseRef-Public-Domain, approved, CQ21976 +maven/mavencentral/org.bouncycastle/bcprov-jdk15on/1.65.01, MIT AND LicenseRef-Public-Domain, approved, CQ21977 +maven/mavencentral/org.eclipse.jetty/jetty-http/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-io/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-security/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-server/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-servlet/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-util-ajax/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jetty/jetty-util/9.4.36.v20210114, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ant.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ant/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.archive/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.gpg.bc/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.http.apache/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.http.server/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.http.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.junit.http/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.junit.ssh/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.junit/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs.server.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs.server/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.lfs/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.pgm.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.pgm/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ssh.apache.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ssh.apache/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ssh.jsch/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.test/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit.ui/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit/5.11.0-SNAPSHOT, , approved, eclipse +maven/mavencentral/org.hamcrest/hamcrest-core/1.3, BSD-2-Clause, approved, CQ7063 +maven/mavencentral/org.mockito/mockito-core/2.23.0, MIT, approved, CQ17976 +maven/mavencentral/org.objenesis/objenesis/2.6, Apache-2.0, approved, CQ15478 +maven/mavencentral/org.openjdk.jmh/jmh-core/1.21, , restricted, CQ20517 +maven/mavencentral/org.openjdk.jmh/jmh-generator-annprocess/1.21, , restricted, CQ20518 +maven/mavencentral/org.osgi/org.osgi.core/4.3.1, Apache-2.0, approved, CQ10111 +maven/mavencentral/org.slf4j/slf4j-api/1.7.30, MIT, approved, CQ13368 +maven/mavencentral/org.slf4j/slf4j-log4j12/1.7.30, MIT, approved, CQ7665 +maven/mavencentral/org.tukaani/xz/1.8, LicenseRef-Public-Domain, approved, CQ15386 From 7a3f62818078dea867fe7c5b4e28d215821086b6 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 7 Mar 2021 18:41:05 +0100 Subject: [PATCH 11/11] Manually set status of jmh dependencies The following jmh dependencies were approved as works-with: - jmh-core/1.21 has GPL-2.0 license and was approved in CQ20517 - jmh-generator-annprocess/1.21 has GPL-2.0 license and was approved in CQ20518 Change-Id: Ibbe28c6e8359c576b23f40281e74f2e0d4a0dee0 --- DEPENDENCIES | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index 541a999de..bffe3d928 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -58,8 +58,8 @@ maven/mavencentral/org.eclipse.jgit/org.eclipse.jgit/5.11.0-SNAPSHOT, , approved maven/mavencentral/org.hamcrest/hamcrest-core/1.3, BSD-2-Clause, approved, CQ7063 maven/mavencentral/org.mockito/mockito-core/2.23.0, MIT, approved, CQ17976 maven/mavencentral/org.objenesis/objenesis/2.6, Apache-2.0, approved, CQ15478 -maven/mavencentral/org.openjdk.jmh/jmh-core/1.21, , restricted, CQ20517 -maven/mavencentral/org.openjdk.jmh/jmh-generator-annprocess/1.21, , restricted, CQ20518 +maven/mavencentral/org.openjdk.jmh/jmh-core/1.21, GPL-2.0, approved, CQ20517 +maven/mavencentral/org.openjdk.jmh/jmh-generator-annprocess/1.21, GPL-2.0, approved, CQ20518 maven/mavencentral/org.osgi/org.osgi.core/4.3.1, Apache-2.0, approved, CQ10111 maven/mavencentral/org.slf4j/slf4j-api/1.7.30, MIT, approved, CQ13368 maven/mavencentral/org.slf4j/slf4j-log4j12/1.7.30, MIT, approved, CQ7665