diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java new file mode 100644 index 000000000..17f04c854 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -0,0 +1,539 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toList; + +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheBuilder; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.util.IO; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +@SuppressWarnings("boxing") +public class PackInserterTest extends RepositoryTestCase { + private WindowCacheConfig origWindowCacheConfig; + + @Before + public void setWindowCacheConfig() { + origWindowCacheConfig = new WindowCacheConfig(); + origWindowCacheConfig.install(); + } + + @After + public void resetWindowCacheConfig() { + origWindowCacheConfig.install(); + } + + @Before + public void emptyAtSetUp() throws Exception { + assertEquals(0, listPacks().size()); + assertNoObjects(); + } + + @Test + public void noFlush() throws Exception { + try (PackInserter ins = newInserter()) { + ins.insert(OBJ_BLOB, Constants.encode("foo contents")); + // No flush. + } + assertNoObjects(); + } + + @Test + public void flushEmptyPack() throws Exception { + try (PackInserter ins = newInserter()) { + ins.flush(); + } + assertNoObjects(); + } + + @Test + public void singlePack() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + ObjectId treeId; + ObjectId commitId; + byte[] commit; + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + + DirCache dc = DirCache.newInCore(); + DirCacheBuilder b = dc.builder(); + DirCacheEntry dce = new DirCacheEntry("foo"); + dce.setFileMode(FileMode.REGULAR_FILE); + dce.setObjectId(blobId); + b.add(dce); + b.finish(); + treeId = dc.writeTree(ins); + + CommitBuilder cb = new CommitBuilder(); + cb.setTreeId(treeId); + cb.setAuthor(author); + cb.setCommitter(committer); + cb.setMessage("Commit message"); + commit = cb.toByteArray(); + commitId = ins.insert(cb); + ins.flush(); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(1, packs.size()); + assertEquals(3, packs.get(0).getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId, blob); + + CanonicalTreeParser treeParser = + new CanonicalTreeParser(null, reader, treeId); + assertEquals("foo", treeParser.getEntryPathString()); + assertEquals(blobId, treeParser.getEntryObjectId()); + + ObjectLoader commitLoader = reader.open(commitId); + assertEquals(OBJ_COMMIT, commitLoader.getType()); + assertArrayEquals(commit, commitLoader.getBytes()); + } + } + + @Test + public void multiplePacks() throws Exception { + ObjectId blobId1; + ObjectId blobId2; + byte[] blob1 = Constants.encode("blob1"); + byte[] blob2 = Constants.encode("blob2"); + + try (PackInserter ins = newInserter()) { + blobId1 = ins.insert(OBJ_BLOB, blob1); + ins.flush(); + blobId2 = ins.insert(OBJ_BLOB, blob2); + ins.flush(); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(2, packs.size()); + assertEquals(1, packs.get(0).getObjectCount()); + assertEquals(1, packs.get(1).getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId1, blob1); + assertBlob(reader, blobId2, blob2); + } + } + + @Test + public void largeBlob() throws Exception { + ObjectId blobId; + byte[] blob = newLargeBlob(); + try (PackInserter ins = newInserter()) { + assertThat(blob.length, greaterThan(ins.getBufferSize())); + blobId = + ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)); + ins.flush(); + } + + assertPacksOnly(); + Collection packs = listPacks(); + assertEquals(1, packs.size()); + PackFile p = packs.iterator().next(); + assertEquals(1, p.getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId, blob); + } + } + + @Test + public void overwriteExistingPack() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + ins.flush(); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(1, packs.size()); + PackFile pack = packs.get(0); + assertEquals(1, pack.getObjectCount()); + + String inode = getInode(pack.getPackFile()); + + try (PackInserter ins = newInserter()) { + ins.checkExisting(false); + assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); + ins.flush(); + } + + assertPacksOnly(); + packs = listPacks(); + assertEquals(1, packs.size()); + pack = packs.get(0); + assertEquals(1, pack.getObjectCount()); + + if (inode != null) { + // Old file was overwritten with new file, although objects were + // equivalent. + assertNotEquals(inode, getInode(pack.getPackFile())); + } + } + + @Test + public void checkExisting() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + ins.insert(OBJ_BLOB, Constants.encode("another blob")); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + ins.checkExisting(false); + assertEquals(blobId, ins.insert(OBJ_BLOB, blob)); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(2, listPacks().size()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId, blob); + } + } + + @Test + public void insertSmallInputStreamRespectsCheckExisting() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + try (PackInserter ins = newInserter()) { + assertThat(blob.length, lessThan(ins.getBufferSize())); + blobId = ins.insert(OBJ_BLOB, blob); + ins.insert(OBJ_BLOB, Constants.encode("another blob")); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + assertEquals(blobId, + ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob))); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + } + + @Test + public void insertLargeInputStreamBypassesCheckExisting() throws Exception { + ObjectId blobId; + byte[] blob = newLargeBlob(); + + try (PackInserter ins = newInserter()) { + assertThat(blob.length, greaterThan(ins.getBufferSize())); + blobId = ins.insert(OBJ_BLOB, blob); + ins.insert(OBJ_BLOB, Constants.encode("another blob")); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(1, listPacks().size()); + + try (PackInserter ins = newInserter()) { + assertEquals(blobId, + ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob))); + ins.flush(); + } + + assertPacksOnly(); + assertEquals(2, listPacks().size()); + } + + @Test + public void readBackSmallFiles() throws Exception { + ObjectId blobId1; + ObjectId blobId2; + ObjectId blobId3; + byte[] blob1 = Constants.encode("blob1"); + byte[] blob2 = Constants.encode("blob2"); + byte[] blob3 = Constants.encode("blob3"); + try (PackInserter ins = newInserter()) { + assertThat(blob1.length, lessThan(ins.getBufferSize())); + blobId1 = ins.insert(OBJ_BLOB, blob1); + + try (ObjectReader reader = ins.newReader()) { + assertBlob(reader, blobId1, blob1); + } + + // Read-back should not mess up the file pointer. + blobId2 = ins.insert(OBJ_BLOB, blob2); + ins.flush(); + + blobId3 = ins.insert(OBJ_BLOB, blob3); + } + + assertPacksOnly(); + List packs = listPacks(); + assertEquals(1, packs.size()); + assertEquals(2, packs.get(0).getObjectCount()); + + try (ObjectReader reader = db.newObjectReader()) { + assertBlob(reader, blobId1, blob1); + assertBlob(reader, blobId2, blob2); + + try { + reader.open(blobId3); + fail("Expected MissingObjectException"); + } catch (MissingObjectException expected) { + // Expected. + } + } + } + + @Test + public void readBackLargeFile() throws Exception { + ObjectId blobId; + byte[] blob = newLargeBlob(); + + WindowCacheConfig wcc = new WindowCacheConfig(); + wcc.setStreamFileThreshold(1024); + wcc.install(); + try (ObjectReader reader = db.newObjectReader()) { + assertThat(blob.length, greaterThan(reader.getStreamFileThreshold())); + } + + try (PackInserter ins = newInserter()) { + blobId = ins.insert(OBJ_BLOB, blob); + + try (ObjectReader reader = ins.newReader()) { + // Double-check threshold is propagated. + assertThat(blob.length, greaterThan(reader.getStreamFileThreshold())); + assertBlob(reader, blobId, blob); + } + } + + assertPacksOnly(); + // Pack was streamed out to disk and read back from the temp file, but + // ultimately rolled back and deleted. + assertEquals(0, listPacks().size()); + + try (ObjectReader reader = db.newObjectReader()) { + try { + reader.open(blobId); + fail("Expected MissingObjectException"); + } catch (MissingObjectException expected) { + // Expected. + } + } + } + + @Test + public void readBackFallsBackToRepo() throws Exception { + ObjectId blobId; + byte[] blob = Constants.encode("foo contents"); + try (PackInserter ins = newInserter()) { + assertThat(blob.length, lessThan(ins.getBufferSize())); + blobId = ins.insert(OBJ_BLOB, blob); + ins.flush(); + } + + try (PackInserter ins = newInserter(); + ObjectReader reader = ins.newReader()) { + assertBlob(reader, blobId, blob); + } + } + + private List listPacks() throws Exception { + List fromOpenDb = listPacks(db); + List reopened; + try (FileRepository db2 = new FileRepository(db.getDirectory())) { + reopened = listPacks(db2); + } + assertEquals(fromOpenDb.size(), reopened.size()); + for (int i = 0 ; i < fromOpenDb.size(); i++) { + PackFile a = fromOpenDb.get(i); + PackFile b = reopened.get(i); + assertEquals(a.getPackName(), b.getPackName()); + assertEquals( + a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); + assertEquals(a.getObjectCount(), b.getObjectCount()); + a.getObjectCount(); + } + return fromOpenDb; + } + + private static List listPacks(FileRepository db) throws Exception { + return db.getObjectDatabase().getPacks().stream() + .sorted(comparing(PackFile::getPackName)).collect(toList()); + } + + private PackInserter newInserter() { + return db.getObjectDatabase().newPackInserter(); + } + + private static byte[] newLargeBlob() { + byte[] blob = new byte[10240]; + for (int i = 0; i < blob.length; i++) { + blob[i] = (byte) ('0' + (i % 10)); + } + return blob; + } + + private static String getInode(File f) throws Exception { + BasicFileAttributes attrs = Files.readAttributes( + f.toPath(), BasicFileAttributes.class); + Object k = attrs.fileKey(); + if (k == null) { + return null; + } + Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$"); + Matcher m = p.matcher(k.toString()); + return m.matches() ? m.group(1) : null; + } + + private static void assertBlob(ObjectReader reader, ObjectId id, + byte[] expected) throws Exception { + ObjectLoader loader = reader.open(id); + assertEquals(OBJ_BLOB, loader.getType()); + assertEquals(expected.length, loader.getSize()); + try (ObjectStream s = loader.openStream()) { + int n = (int) s.getSize(); + byte[] actual = new byte[n]; + assertEquals(n, IO.readFully(s, actual, 0)); + assertArrayEquals(expected, actual); + } + } + + private void assertPacksOnly() throws Exception { + new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx")) + .assertNoBadFiles(db.getObjectDatabase().getDirectory()); + } + + private void assertNoObjects() throws Exception { + new BadFileCollector(f -> true) + .assertNoBadFiles(db.getObjectDatabase().getDirectory()); + } + + private static class BadFileCollector extends SimpleFileVisitor { + private final Predicate badName; + private List bad; + + BadFileCollector(Predicate badName) { + this.badName = badName; + } + + void assertNoBadFiles(File f) throws IOException { + bad = new ArrayList<>(); + Files.walkFileTree(f.toPath(), this); + if (!bad.isEmpty()) { + fail("unexpected files in object directory: " + bad); + } + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + String name = file.getFileName().toString(); + if (!attrs.isDirectory() && badName.test(name)) { + bad.add(name); + } + return FileVisitResult.CONTINUE; + } + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index f586aee93..d598f7923 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -90,6 +90,7 @@ cannotParseDate=The date specification "{0}" could not be parsed with the follow cannotParseGitURIish=Cannot parse Git URI-ish cannotPullOnARepoWithState=Cannot pull into a repository with state: {0} cannotRead=Cannot read {0} +cannotReadBackDelta=Cannot read delta type {0} cannotReadBlob=Cannot read blob {0} cannotReadCommit=Cannot read commit {0} cannotReadFile=Cannot read file {0} @@ -687,6 +688,7 @@ unencodeableFile=Unencodable file: {0} unexpectedCompareResult=Unexpected metadata comparison result: {0} unexpectedEndOfConfigFile=Unexpected end of config file unexpectedEndOfInput=Unexpected end of input +unexpectedEofInPack=Unexpected EOF in partially created pack unexpectedHunkTrailer=Unexpected hunk trailer unexpectedOddResult=odd: {0} + {1} - {2} unexpectedRefReport={0}: unexpected ref report: {1} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties index 4bbc4cc71..2c4bd06a3 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/storage/dfs/DfsText.properties @@ -1,6 +1,4 @@ cannotReadIndex=Cannot read index {0} -cannotReadBackDelta=Cannot read delta type {0} shortReadOfBlock=Short read of block at {0} in pack {1}; expected {2} bytes, received only {3} shortReadOfIndex=Short read of index {0} -unexpectedEofInPack=Unexpected EOF in partially created pack willNotStoreEmptyPack=Cannot store empty pack diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index c41fc51bd..cc685e44d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -149,6 +149,7 @@ public static JGitText get() { /***/ public String cannotParseGitURIish; /***/ public String cannotPullOnARepoWithState; /***/ public String cannotRead; + /***/ public String cannotReadBackDelta; /***/ public String cannotReadBlob; /***/ public String cannotReadCommit; /***/ public String cannotReadFile; @@ -746,6 +747,7 @@ public static JGitText get() { /***/ public String unexpectedCompareResult; /***/ public String unexpectedEndOfConfigFile; /***/ public String unexpectedEndOfInput; + /***/ public String unexpectedEofInPack; /***/ public String unexpectedHunkTrailer; /***/ public String unexpectedOddResult; /***/ public String unexpectedRefReport; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index 01654d45d..19e86522c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -500,7 +500,7 @@ private int setInput(long pos, Inflater inf) inf.setInput(currBuf, s, n); return n; } - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); } private DfsBlock getOrLoadBlock(long pos) throws IOException { @@ -513,7 +513,7 @@ private DfsBlock getOrLoadBlock(long pos) throws IOException { for (int p = 0; p < blockSize;) { int n = out.read(s + p, ByteBuffer.wrap(d, p, blockSize - p)); if (n <= 0) - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); p += n; } b = new DfsBlock(packKey, s, d); @@ -569,13 +569,13 @@ public ObjectLoader open(AnyObjectId objectId, int typeHint) byte[] buf = buffer(); int cnt = packOut.read(obj.getOffset(), buf, 0, 20); if (cnt <= 0) - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); int c = buf[0] & 0xff; int type = (c >> 4) & 7; if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) throw new IOException(MessageFormat.format( - DfsText.get().cannotReadBackDelta, Integer.toString(type))); + JGitText.get().cannotReadBackDelta, Integer.toString(type))); if (typeHint != OBJ_ANY && type != typeHint) { throw new IncorrectObjectTypeException(objectId.copy(), typeHint); } @@ -585,7 +585,7 @@ public ObjectLoader open(AnyObjectId objectId, int typeHint) int shift = 4; while ((c & 0x80) != 0) { if (ptr >= cnt) - throw new EOFException(DfsText.get().unexpectedEofInPack); + throw new EOFException(JGitText.get().unexpectedEofInPack); c = buf[ptr++] & 0xff; sz += ((long) (c & 0x7f)) << shift; shift += 7; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java index 862454759..dedcab04d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsText.java @@ -55,9 +55,7 @@ public static DfsText get() { // @formatter:off /***/ public String cannotReadIndex; - /***/ public String cannotReadBackDelta; /***/ public String shortReadOfBlock; /***/ public String shortReadOfIndex; - /***/ public String unexpectedEofInPack; /***/ public String willNotStoreEmptyPack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 8cd8ee152..f992a3340 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1173,16 +1173,7 @@ public int compare(PackExt o1, PackExt o2) { // rename the temporary files to real files File realPack = nameFor(id, ".pack"); //$NON-NLS-1$ - // if the packfile already exists (because we are rewriting a - // packfile for the same set of objects maybe with different - // PackConfig) then make sure we get rid of all handles on the file. - // Windows will not allow for rename otherwise. - if (realPack.exists()) - for (PackFile p : repo.getObjectDatabase().getPacks()) - if (realPack.getPath().equals(p.getPackFile().getPath())) { - p.close(); - break; - } + repo.getObjectDatabase().closeAllPackHandles(realPack); tmpPack.setReadOnly(); FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index d953b87c4..153c7dd92 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -220,6 +220,16 @@ public ObjectDirectoryInserter newInserter() { return new ObjectDirectoryInserter(this, config); } + /** + * Create a new inserter that inserts all objects as pack files, not loose + * objects. + * + * @return new inserter. + */ + public PackInserter newPackInserter() { + return new PackInserter(this); + } + @Override public void close() { unpackedObjectCache.clear(); @@ -814,8 +824,6 @@ private void insertPack(final PackFile pf) { final PackFile[] oldList = o.packs; final String name = pf.getPackFile().getName(); for (PackFile p : oldList) { - if (PackFile.SORT.compare(pf, p) < 0) - break; if (name.equals(p.getPackFile().getName())) return; } @@ -971,6 +979,21 @@ private Set listPackDirectory() { return nameSet; } + void closeAllPackHandles(File packFile) { + // if the packfile already exists (because we are rewriting a + // packfile for the same set of objects maybe with different + // PackConfig) then make sure we get rid of all handles on the file. + // Windows will not allow for rename otherwise. + if (packFile.exists()) { + for (PackFile p : getPacks()) { + if (packFile.getPath().equals(p.getPackFile().getPath())) { + p.close(); + break; + } + } + } + } + AlternateHandle[] myAlternates() { AlternateHandle[] alt = alternates.get(); if (alt == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java new file mode 100644 index 000000000..dd83e251f --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -0,0 +1,639 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; +import static org.eclipse.jgit.lib.Constants.OBJ_OFS_DELTA; +import static org.eclipse.jgit.lib.Constants.OBJ_REF_DELTA; + +import java.io.BufferedInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.channels.Channels; +import java.text.MessageFormat; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.zip.CRC32; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.InflaterCache; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.transport.PackParser; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.BlockList; +import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.io.CountingOutputStream; +import org.eclipse.jgit.util.sha1.SHA1; + +/** + * Object inserter that inserts one pack per call to {@link #flush()}, and never + * inserts loose objects. + */ +class PackInserter extends ObjectInserter { + /** Always produce version 2 indexes, to get CRC data. */ + private static final int INDEX_VERSION = 2; + + private final ObjectDirectory db; + + private List objectList; + private ObjectIdOwnerMap objectMap; + private boolean rollback; + private boolean checkExisting = true; + + private int compression = Deflater.BEST_COMPRESSION; + private File tmpPack; + private PackStream packOut; + private Inflater cachedInflater; + + PackInserter(ObjectDirectory db) { + this.db = db; + } + + /** + * @param check + * if false, will write out possibly-duplicate objects without + * first checking whether they exist in the repo; default is true. + */ + public void checkExisting(boolean check) { + checkExisting = check; + } + + /** + * @param compression + * compression level for zlib deflater. + */ + public void setCompressionLevel(int compression) { + this.compression = compression; + } + + int getBufferSize() { + return buffer().length; + } + + @Override + public ObjectId insert(int type, byte[] data, int off, int len) + throws IOException { + ObjectId id = idFor(type, data, off, len); + if (objectMap != null && objectMap.contains(id)) { + return id; + } + // Ignore loose objects, which are potentially unreachable. + if (checkExisting && db.hasPackedObject(id)) { + return id; + } + + long offset = beginObject(type, len); + packOut.compress.write(data, off, len); + packOut.compress.finish(); + return endObject(id, offset); + } + + @Override + public ObjectId insert(int type, long len, InputStream in) + throws IOException { + byte[] buf = buffer(); + if (len <= buf.length) { + IO.readFully(in, buf, 0, (int) len); + return insert(type, buf, 0, (int) len); + } + + long offset = beginObject(type, len); + SHA1 md = digest(); + md.update(Constants.encodedTypeString(type)); + md.update((byte) ' '); + md.update(Constants.encodeASCII(len)); + md.update((byte) 0); + + while (0 < len) { + int n = in.read(buf, 0, (int) Math.min(buf.length, len)); + if (n <= 0) { + throw new EOFException(); + } + md.update(buf, 0, n); + packOut.compress.write(buf, 0, n); + len -= n; + } + packOut.compress.finish(); + return endObject(md.toObjectId(), offset); + } + + private long beginObject(int type, long len) throws IOException { + if (packOut == null) { + beginPack(); + } + long offset = packOut.getOffset(); + packOut.beginObject(type, len); + return offset; + } + + private ObjectId endObject(ObjectId id, long offset) { + PackedObjectInfo obj = new PackedObjectInfo(id); + obj.setOffset(offset); + obj.setCRC((int) packOut.crc32.getValue()); + objectList.add(obj); + objectMap.addIfAbsent(obj); + return id; + } + + private static File idxFor(File packFile) { + String p = packFile.getName(); + return new File( + packFile.getParentFile(), + p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$ + } + + private void beginPack() throws IOException { + objectList = new BlockList<>(); + objectMap = new ObjectIdOwnerMap<>(); + + rollback = true; + tmpPack = File.createTempFile("insert_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$ + packOut = new PackStream(tmpPack); + + // Write the header as though it were a single object pack. + packOut.write(packOut.hdrBuf, 0, writePackHeader(packOut.hdrBuf, 1)); + } + + private static int writePackHeader(byte[] buf, int objectCount) { + System.arraycopy(Constants.PACK_SIGNATURE, 0, buf, 0, 4); + NB.encodeInt32(buf, 4, 2); // Always use pack version 2. + NB.encodeInt32(buf, 8, objectCount); + return 12; + } + + @Override + public PackParser newPackParser(InputStream in) { + throw new UnsupportedOperationException(); + } + + @Override + public ObjectReader newReader() { + return new Reader(); + } + + @Override + public void flush() throws IOException { + if (tmpPack == null) { + return; + } + + if (packOut == null) { + throw new IOException(); + } + + byte[] packHash; + try { + packHash = packOut.finishPack(); + } finally { + packOut.close(); + packOut = null; + } + + Collections.sort(objectList); + File tmpIdx = idxFor(tmpPack); + writePackIndex(tmpIdx, packHash, objectList); + + File realPack = new File( + new File(db.getDirectory(), "pack"), //$NON-NLS-1$ + "pack-" + computeName(objectList).name() + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ + db.closeAllPackHandles(realPack); + tmpPack.setReadOnly(); + FileUtils.rename(tmpPack, realPack, ATOMIC_MOVE); + + File realIdx = idxFor(realPack); + tmpIdx.setReadOnly(); + try { + FileUtils.rename(tmpIdx, realIdx, ATOMIC_MOVE); + } catch (IOException e) { + File newIdx = new File( + realIdx.getParentFile(), realIdx.getName() + ".new"); //$NON-NLS-1$ + try { + FileUtils.rename(tmpIdx, newIdx, ATOMIC_MOVE); + } catch (IOException e2) { + newIdx = tmpIdx; + e = e2; + } + throw new IOException(MessageFormat.format( + JGitText.get().panicCantRenameIndexFile, newIdx, + realIdx), e); + } + + db.openPack(realPack); + rollback = false; + clear(); + } + + private static void writePackIndex(File idx, byte[] packHash, + List list) throws IOException { + try (OutputStream os = new FileOutputStream(idx)) { + PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION); + w.write(list, packHash); + } + } + + private ObjectId computeName(List list) { + SHA1 md = digest().reset(); + byte[] buf = buffer(); + for (PackedObjectInfo otp : list) { + otp.copyRawTo(buf, 0); + md.update(buf, 0, OBJECT_ID_LENGTH); + } + return ObjectId.fromRaw(md.digest()); + } + + @Override + public void close() { + try { + if (packOut != null) { + try { + packOut.close(); + } catch (IOException err) { + // Ignore a close failure, the pack should be removed. + } + } + if (rollback && tmpPack != null) { + try { + FileUtils.delete(tmpPack); + } catch (IOException e) { + // Still delete idx. + } + try { + FileUtils.delete(idxFor(tmpPack)); + } catch (IOException e) { + // Ignore error deleting temp idx. + } + rollback = false; + } + } finally { + clear(); + try { + InflaterCache.release(cachedInflater); + } finally { + cachedInflater = null; + } + } + } + + private void clear() { + objectList = null; + objectMap = null; + tmpPack = null; + packOut = null; + } + + private Inflater inflater() { + if (cachedInflater == null) { + cachedInflater = InflaterCache.get(); + } else { + cachedInflater.reset(); + } + return cachedInflater; + } + + private class PackStream extends OutputStream { + final byte[] hdrBuf; + final CRC32 crc32; + final DeflaterOutputStream compress; + + private final RandomAccessFile file; + private final CountingOutputStream out; + private final Deflater deflater; + + PackStream(File pack) throws IOException { + file = new RandomAccessFile(pack, "rw"); //$NON-NLS-1$ + out = new CountingOutputStream(new FileOutputStream(file.getFD())); + deflater = new Deflater(compression); + compress = new DeflaterOutputStream(this, deflater, 8192); + hdrBuf = new byte[32]; + crc32 = new CRC32(); + } + + long getOffset() { + return out.getCount(); + } + + void beginObject(int objectType, long length) throws IOException { + crc32.reset(); + deflater.reset(); + write(hdrBuf, 0, encodeTypeSize(objectType, length)); + } + + private int encodeTypeSize(int type, long rawLength) { + long nextLength = rawLength >>> 4; + hdrBuf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (rawLength & 0x0F)); + rawLength = nextLength; + int n = 1; + while (rawLength > 0) { + nextLength >>>= 7; + hdrBuf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (rawLength & 0x7F)); + rawLength = nextLength; + } + return n; + } + + @Override + public void write(final int b) throws IOException { + hdrBuf[0] = (byte) b; + write(hdrBuf, 0, 1); + } + + @Override + public void write(byte[] data, int off, int len) throws IOException { + crc32.update(data, off, len); + out.write(data, off, len); + } + + byte[] finishPack() throws IOException { + // Overwrite placeholder header with actual object count, then hash. + file.seek(0); + write(hdrBuf, 0, writePackHeader(hdrBuf, objectList.size())); + + byte[] buf = buffer(); + SHA1 md = digest().reset(); + file.seek(0); + while (true) { + int r = file.read(buf); + if (r < 0) { + break; + } + md.update(buf, 0, r); + } + byte[] packHash = md.digest(); + out.write(packHash, 0, packHash.length); + return packHash; + } + + @Override + public void close() throws IOException { + deflater.end(); + out.close(); + file.close(); + } + + byte[] inflate(long filePos, int len) throws IOException, DataFormatException { + byte[] dstbuf; + try { + dstbuf = new byte[len]; + } catch (OutOfMemoryError noMemory) { + return null; // Caller will switch to large object streaming. + } + + byte[] srcbuf = buffer(); + Inflater inf = inflater(); + filePos += setInput(filePos, inf, srcbuf); + for (int dstoff = 0;;) { + int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff); + dstoff += n; + if (inf.finished()) { + return dstbuf; + } + if (inf.needsInput()) { + filePos += setInput(filePos, inf, srcbuf); + } else if (n == 0) { + throw new DataFormatException(); + } + } + } + + private int setInput(long filePos, Inflater inf, byte[] buf) + throws IOException { + if (file.getFilePointer() != filePos) { + file.seek(filePos); + } + int n = file.read(buf); + if (n < 0) { + throw new EOFException(JGitText.get().unexpectedEofInPack); + } + inf.setInput(buf, 0, n); + return n; + } + } + + private class Reader extends ObjectReader { + private final ObjectReader ctx; + + private Reader() { + ctx = db.newReader(); + setStreamFileThreshold(ctx.getStreamFileThreshold()); + } + + @Override + public ObjectReader newReader() { + return db.newReader(); + } + + @Override + public ObjectInserter getCreatedFromInserter() { + return PackInserter.this; + } + + @Override + public Collection resolve(AbbreviatedObjectId id) + throws IOException { + Collection stored = ctx.resolve(id); + if (objectList == null) { + return stored; + } + + Set r = new HashSet<>(stored.size() + 2); + r.addAll(stored); + for (PackedObjectInfo obj : objectList) { + if (id.prefixCompare(obj) == 0) { + r.add(obj.copy()); + } + } + return r; + } + + @Override + public ObjectLoader open(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + if (objectMap == null) { + return ctx.open(objectId, typeHint); + } + + PackedObjectInfo obj = objectMap.get(objectId); + if (obj == null) { + return ctx.open(objectId, typeHint); + } + + byte[] buf = buffer(); + RandomAccessFile f = packOut.file; + f.seek(obj.getOffset()); + int cnt = f.read(buf, 0, 20); + if (cnt <= 0) { + throw new EOFException(JGitText.get().unexpectedEofInPack); + } + + int c = buf[0] & 0xff; + int type = (c >> 4) & 7; + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + throw new IOException(MessageFormat.format( + JGitText.get().cannotReadBackDelta, Integer.toString(type))); + } + if (typeHint != OBJ_ANY && type != typeHint) { + throw new IncorrectObjectTypeException(objectId.copy(), typeHint); + } + + long sz = c & 0x0f; + int ptr = 1; + int shift = 4; + while ((c & 0x80) != 0) { + if (ptr >= cnt) { + throw new EOFException(JGitText.get().unexpectedEofInPack); + } + c = buf[ptr++] & 0xff; + sz += ((long) (c & 0x7f)) << shift; + shift += 7; + } + + long zpos = obj.getOffset() + ptr; + if (sz < getStreamFileThreshold()) { + byte[] data = inflate(obj, zpos, (int) sz); + if (data != null) { + return new ObjectLoader.SmallObject(type, data); + } + } + return new StreamLoader(f, type, sz, zpos); + } + + private byte[] inflate(PackedObjectInfo obj, long zpos, int sz) + throws IOException, CorruptObjectException { + try { + return packOut.inflate(zpos, sz); + } catch (DataFormatException dfe) { + CorruptObjectException coe = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(obj.getOffset()), + tmpPack.getAbsolutePath())); + coe.initCause(dfe); + throw coe; + } + } + + @Override + public Set getShallowCommits() throws IOException { + return ctx.getShallowCommits(); + } + + @Override + public void close() { + ctx.close(); + } + + private class StreamLoader extends ObjectLoader { + private final RandomAccessFile file; + private final int type; + private final long size; + private final long pos; + + StreamLoader(RandomAccessFile file, int type, long size, long pos) { + this.file = file; + this.type = type; + this.size = size; + this.pos = pos; + } + + @Override + public ObjectStream openStream() + throws MissingObjectException, IOException { + int bufsz = buffer().length; + file.seek(pos); + return new ObjectStream.Filter( + type, size, + new BufferedInputStream( + new InflaterInputStream( + Channels.newInputStream(packOut.file.getChannel()), + inflater(), bufsz), + bufsz)); + } + + @Override + public int getType() { + return type; + } + + @Override + public long getSize() { + return size; + } + + @Override + public byte[] getCachedBytes() throws LargeObjectException { + throw new LargeObjectException.ExceedsLimit( + getStreamFileThreshold(), size); + } + } + } +}