From d2846cc8b2a831a089ee768a0475e64ec5b85519 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Tue, 2 Mar 2021 09:53:08 +0100 Subject: [PATCH 1/8] ApplyCommand: convert to git internal format before applying patch Applying a patch on Windows failed if the patch had the (normal) single-LF line endings, but the file on disk had the usual Windows CR-LF line endings. Git (and JGit) compute diffs on the git-internal blob, i.e., after CR-LF transformation and clean filtering. Applying patches to files directly is thus incorrect and may fail if CR-LF settings don't match, or if clean/smudge filtering is involved. Change ApplyCommand to run the file content through the check-in filters before applying the patch, and run the result through the check-out filters. This makes patch application succeed even if the patch has single-LFs, but the file has CR-LF and core.autocrlf is true. Add tests for various combinations of line endings in the file and in the patch, and a test to verify the clean/smudge handling. See also [1]. Running the file though clean/smudge may give strange results with LFS-managed files. JGit's DiffFormatter has some extra code and applies the smudge filter again after having run the file through the check-in filters (CR-LF and clean). So JGit can actually produce a diff on LFS-managed files using the normal diff machinery. (If it doesn't run out of memory, that is. After all, LFS is intended for _large_ files.) How such a diff would be applied with either C git or JGit is entirely unclear; neither has any code for this special case. Compare also [2]. Note that C git just doesn't know about LFS and always diffs after the check-in filter chain, so for LFS files, it'll produce a diff of the LFS pointers. [1] https://github.com/git/git/commit/c24f3abac [2] https://github.com/git-lfs/git-lfs/issues/440 Bug: 571585 Change-Id: I8f71ff26313b5773ff1da612b0938ad2f18751f5 Signed-off-by: Thomas Wolf --- .../tst-rsrc/org/eclipse/jgit/diff/crlf.patch | Bin 0 -> 113 bytes .../org/eclipse/jgit/diff/crlf2.patch | Bin 0 -> 121 bytes .../org/eclipse/jgit/diff/crlf2_PostImage | Bin 0 -> 15 bytes .../org/eclipse/jgit/diff/crlf2_PreImage | Bin 0 -> 15 bytes .../org/eclipse/jgit/diff/crlf3.patch | Bin 0 -> 112 bytes .../org/eclipse/jgit/diff/crlf3_PostImage | Bin 0 -> 15 bytes .../org/eclipse/jgit/diff/crlf3_PreImage | Bin .../org/eclipse/jgit/diff/crlf4.patch | Bin 0 -> 128 bytes .../org/eclipse/jgit/diff/crlf4_PostImage | Bin 0 -> 15 bytes .../org/eclipse/jgit/diff/crlf_PostImage | Bin 0 -> 15 bytes .../org/eclipse/jgit/diff/crlf_PreImage | Bin 0 -> 15 bytes .../org/eclipse/jgit/diff/smudgetest.patch | Bin 0 -> 146 bytes .../eclipse/jgit/diff/smudgetest_PostImage | Bin 0 -> 19 bytes .../org/eclipse/jgit/diff/smudgetest_PreImage | Bin 0 -> 18 bytes .../eclipse/jgit/api/ApplyCommandTest.java | 191 +++++++++++- .../org/eclipse/jgit/api/ApplyCommand.java | 279 ++++++++++++++++-- 16 files changed, 445 insertions(+), 25 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf2.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf2_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf2_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf4.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf4_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest_PreImage diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf.patch new file mode 100644 index 0000000000000000000000000000000000000000..01eb0b9510e6cd8f89e7889974c5cec9abdae454 GIT binary patch literal 113 zcmXwwOA3H63k2F-1NGf@>oLOx44fVW literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf2.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf2.patch new file mode 100644 index 0000000000000000000000000000000000000000..5a6210489c0c83d3776b53360f488d18694c5655 GIT binary patch literal 121 zcmX|&%L>9U5CHEB`G>u)n=Nffxa03lG59bOu^7|dD literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3_PostImage new file mode 100644 index 0000000000000000000000000000000000000000..05c1c782e5f2a67daf260890dc9251a02272497a GIT binary patch literal 15 UcmYex&*$Yz%S;6lrMbLZ03&e(1^@s6 literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf3_PreImage new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf4.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/crlf4.patch new file mode 100644 index 0000000000000000000000000000000000000000..0cf606390a9b1a694a4d61340160455d13991c85 GIT binary patch literal 128 zcmX|(!3x4K5C!l174M#Iv(*srmS0h`D+_4?V%5*LP;d@2x2bLh=7+ygxgW- z`hj^TEA;&5f=rKDCUBpR8dm>8z&>6x1uC7Yxe zC>RFVmD=;hMZ)<)Iu;Gm#usAH_44I&&IxD*0hgM3`MbUj?1fRuJ|K~ZLM N2A4u{L26<)7XS}KD;EF& literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest_PostImage new file mode 100644 index 0000000000000000000000000000000000000000..ad630893df6de00ce81f2b40cd5e3e4e05dff2ec GIT binary patch literal 19 YcmWG=4Dxa0DlRC>OwIsOsfpQK06en>EC2ui literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/smudgetest_PreImage new file mode 100644 index 0000000000000000000000000000000000000000..9bbd8c763e9552b54d783dbf3a47f3552cc19920 GIT binary patch literal 18 XcmWG=4Dxa0@^EwllEnq7iP>BLE!hP7 literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java index 055eba718..335a64d70 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2020 IBM Corporation and others + * Copyright (C) 2011, 2021 IBM Corporation and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -18,11 +18,17 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import org.eclipse.jgit.api.errors.PatchApplyException; import org.eclipse.jgit.api.errors.PatchFormatException; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandFactory; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.diff.RawText; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; import org.junit.Test; public class ApplyCommandTest extends RepositoryTestCase { @@ -57,6 +63,189 @@ private ApplyResult init(final String name, final boolean preExists, } } + @Test + public void testCrLf() throws Exception { + try { + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, true); + ApplyResult result = init("crlf", true, true); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), "crlf"), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), "crlf"), + b.getString(0, b.size(), false)); + } finally { + db.getConfig().unset(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF); + } + } + + @Test + public void testCrLfOff() throws Exception { + try { + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, false); + ApplyResult result = init("crlf", true, true); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), "crlf"), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), "crlf"), + b.getString(0, b.size(), false)); + } finally { + db.getConfig().unset(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF); + } + } + + @Test + public void testCrLfEmptyCommitted() throws Exception { + try { + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, true); + ApplyResult result = init("crlf3", true, true); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), "crlf3"), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), "crlf3"), + b.getString(0, b.size(), false)); + } finally { + db.getConfig().unset(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF); + } + } + + @Test + public void testCrLfNewFile() throws Exception { + try { + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, true); + ApplyResult result = init("crlf4", false, true); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), "crlf4"), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), "crlf4"), + b.getString(0, b.size(), false)); + } finally { + db.getConfig().unset(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF); + } + } + + @Test + public void testPatchWithCrLf() throws Exception { + try { + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, false); + ApplyResult result = init("crlf2", true, true); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), "crlf2"), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), "crlf2"), + b.getString(0, b.size(), false)); + } finally { + db.getConfig().unset(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF); + } + } + + @Test + public void testPatchWithCrLf2() throws Exception { + String name = "crlf2"; + try (Git git = new Git(db)) { + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, false); + a = new RawText(readFile(name + "_PreImage")); + write(new File(db.getWorkTree(), name), + a.getString(0, a.size(), false)); + + git.add().addFilepattern(name).call(); + git.commit().setMessage("PreImage").call(); + + b = new RawText(readFile(name + "_PostImage")); + + db.getConfig().setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, true); + ApplyResult result = git.apply() + .setPatch(getTestResource(name + ".patch")).call(); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), name), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), name), + b.getString(0, b.size(), false)); + } finally { + db.getConfig().unset(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF); + } + } + + // Clean/smudge filter for testFiltering. The smudgetest test resources were + // created with C git using a clean filter sed -e "s/A/E/g" and the smudge + // filter sed -e "s/E/A/g". To keep the test independent of the presence of + // sed, implement this with a built-in filter. + private static class ReplaceFilter extends FilterCommand { + + private final char toReplace; + + private final char replacement; + + ReplaceFilter(InputStream in, OutputStream out, char toReplace, + char replacement) { + super(in, out); + this.toReplace = toReplace; + this.replacement = replacement; + } + + @Override + public int run() throws IOException { + int b = in.read(); + if (b < 0) { + in.close(); + out.close(); + return -1; + } + if ((b & 0xFF) == toReplace) { + b = replacement; + } + out.write(b); + return 1; + } + } + + @Test + public void testFiltering() throws Exception { + // Set up filter + FilterCommandFactory clean = (repo, in, out) -> { + return new ReplaceFilter(in, out, 'A', 'E'); + }; + FilterCommandFactory smudge = (repo, in, out) -> { + return new ReplaceFilter(in, out, 'E', 'A'); + }; + FilterCommandRegistry.register("jgit://builtin/a2e/clean", clean); + FilterCommandRegistry.register("jgit://builtin/a2e/smudge", smudge); + try (Git git = new Git(db)) { + Config config = db.getConfig(); + config.setString(ConfigConstants.CONFIG_FILTER_SECTION, "a2e", + "clean", "jgit://builtin/a2e/clean"); + config.setString(ConfigConstants.CONFIG_FILTER_SECTION, "a2e", + "smudge", "jgit://builtin/a2e/smudge"); + write(new File(db.getWorkTree(), ".gitattributes"), + "smudgetest filter=a2e"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("Attributes").call(); + ApplyResult result = init("smudgetest", true, true); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(new File(db.getWorkTree(), "smudgetest"), + result.getUpdatedFiles().get(0)); + checkFile(new File(db.getWorkTree(), "smudgetest"), + b.getString(0, b.size(), false)); + + } finally { + // Tear down filter + FilterCommandRegistry.unregister("jgit://builtin/a2e/clean"); + FilterCommandRegistry.unregister("jgit://builtin/a2e/smudge"); + } + } + @Test public void testAddA1() throws Exception { ApplyResult result = init("A1", false, true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java index e228e8276..5d975ea38 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2020 IBM Corporation and others + * Copyright (C) 2011, 2021 IBM Corporation and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -9,10 +9,16 @@ */ package org.eclipse.jgit.api; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; @@ -20,18 +26,45 @@ import java.util.Iterator; import java.util.List; +import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.PatchApplyException; import org.eclipse.jgit.api.errors.PatchFormatException; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheCheckout; +import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata; +import org.eclipse.jgit.dircache.DirCacheIterator; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.patch.FileHeader; import org.eclipse.jgit.patch.HunkHeader; import org.eclipse.jgit.patch.Patch; +import org.eclipse.jgit.treewalk.FileTreeIterator; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.TreeWalk.OperationType; +import org.eclipse.jgit.treewalk.filter.AndTreeFilter; +import org.eclipse.jgit.treewalk.filter.NotIgnoredFilter; +import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.StringUtils; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.eclipse.jgit.util.FS.ExecutionResult; +import org.eclipse.jgit.util.TemporaryBuffer.LocalFile; +import org.eclipse.jgit.util.io.EolStreamTypeUtil; /** * Apply a patch to files and/or to the index. @@ -45,7 +78,7 @@ public class ApplyCommand extends GitCommand { private InputStream in; /** - * Constructs the command if the patch is to be applied to the index. + * Constructs the command. * * @param repo */ @@ -79,6 +112,7 @@ public ApplyCommand setPatch(InputStream in) { public ApplyResult call() throws GitAPIException, PatchFormatException, PatchApplyException { checkCallable(); + setCallable(false); ApplyResult r = new ApplyResult(); try { final Patch p = new Patch(); @@ -87,19 +121,22 @@ public ApplyResult call() throws GitAPIException, PatchFormatException, } finally { in.close(); } - if (!p.getErrors().isEmpty()) + if (!p.getErrors().isEmpty()) { throw new PatchFormatException(p.getErrors()); + } + Repository repository = getRepository(); + DirCache cache = repository.readDirCache(); for (FileHeader fh : p.getFiles()) { ChangeType type = fh.getChangeType(); File f = null; switch (type) { case ADD: f = getFile(fh.getNewPath(), true); - apply(f, fh); + apply(repository, fh.getNewPath(), cache, f, fh); break; case MODIFY: f = getFile(fh.getOldPath(), false); - apply(f, fh); + apply(repository, fh.getOldPath(), cache, f, fh); break; case DELETE: f = getFile(fh.getOldPath(), false); @@ -118,14 +155,14 @@ public ApplyResult call() throws GitAPIException, PatchFormatException, throw new PatchApplyException(MessageFormat.format( JGitText.get().renameFileFailed, f, dest), e); } - apply(dest, fh); + apply(repository, fh.getOldPath(), cache, dest, fh); break; case COPY: f = getFile(fh.getOldPath(), false); File target = getFile(fh.getNewPath(), false); FileUtils.mkdirs(target.getParentFile(), true); Files.copy(f.toPath(), target.toPath()); - apply(target, fh); + apply(repository, fh.getOldPath(), cache, target, fh); } r.addUpdatedFile(f); } @@ -133,14 +170,13 @@ public ApplyResult call() throws GitAPIException, PatchFormatException, throw new PatchApplyException(MessageFormat.format( JGitText.get().patchApplyException, e.getMessage()), e); } - setCallable(false); return r; } private File getFile(String path, boolean create) throws PatchApplyException { File f = new File(getRepository().getWorkTree(), path); - if (create) + if (create) { try { File parent = f.getParentFile(); FileUtils.mkdirs(parent, true); @@ -149,21 +185,201 @@ private File getFile(String path, boolean create) throw new PatchApplyException(MessageFormat.format( JGitText.get().createNewFileFailed, f), e); } + } return f; } + private void apply(Repository repository, String path, DirCache cache, + File f, FileHeader fh) throws IOException, PatchApplyException { + boolean convertCrLf = needsCrLfConversion(f, fh); + // Use a TreeWalk with a DirCacheIterator to pick up the correct + // clean/smudge filters. CR-LF handling is completely determined by + // whether the file or the patch have CR-LF line endings. + try (TreeWalk walk = new TreeWalk(repository)) { + walk.setOperationType(OperationType.CHECKIN_OP); + FileTreeIterator files = new FileTreeIterator(repository); + int fileIdx = walk.addTree(files); + int cacheIdx = walk.addTree(new DirCacheIterator(cache)); + files.setDirCacheIterator(walk, cacheIdx); + walk.setFilter(AndTreeFilter.create( + PathFilterGroup.createFromStrings(path), + new NotIgnoredFilter(fileIdx))); + walk.setRecursive(true); + if (walk.next()) { + // If the file on disk has no newline characters, convertCrLf + // will be false. In that case we want to honor the normal git + // settings. + EolStreamType streamType = convertCrLf ? EolStreamType.TEXT_CRLF + : walk.getEolStreamType(OperationType.CHECKOUT_OP); + String command = walk.getFilterCommand( + Constants.ATTR_FILTER_TYPE_SMUDGE); + CheckoutMetadata checkOut = new CheckoutMetadata(streamType, command); + FileTreeIterator file = walk.getTree(fileIdx, + FileTreeIterator.class); + if (file != null) { + command = walk + .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN); + RawText raw; + // Can't use file.openEntryStream() as it would do CR-LF + // conversion as usual, not as wanted by us. + try (InputStream input = filterClean(repository, path, + new FileInputStream(f), convertCrLf, command)) { + raw = new RawText(IO.readWholeStream(input, 0).array()); + } + apply(repository, path, raw, f, fh, checkOut); + return; + } + } + } + // File ignored? + RawText raw; + CheckoutMetadata checkOut; + if (convertCrLf) { + try (InputStream input = EolStreamTypeUtil.wrapInputStream( + new FileInputStream(f), EolStreamType.TEXT_LF)) { + raw = new RawText(IO.readWholeStream(input, 0).array()); + } + checkOut = new CheckoutMetadata(EolStreamType.TEXT_CRLF, null); + } else { + raw = new RawText(f); + checkOut = new CheckoutMetadata(EolStreamType.DIRECT, null); + } + apply(repository, path, raw, f, fh, checkOut); + } + + private boolean needsCrLfConversion(File f, FileHeader fileHeader) + throws IOException { + if (!hasCrLf(fileHeader)) { + try (InputStream input = new FileInputStream(f)) { + return RawText.isCrLfText(input); + } + } + return false; + } + + private static boolean hasCrLf(FileHeader fileHeader) { + if (fileHeader == null) { + return false; + } + for (HunkHeader header : fileHeader.getHunks()) { + byte[] buf = header.getBuffer(); + int hunkEnd = header.getEndOffset(); + int lineStart = header.getStartOffset(); + while (lineStart < hunkEnd) { + int nextLineStart = RawParseUtils.nextLF(buf, lineStart); + if (nextLineStart > hunkEnd) { + nextLineStart = hunkEnd; + } + if (nextLineStart <= lineStart) { + break; + } + if (nextLineStart - lineStart > 1) { + char first = (char) (buf[lineStart] & 0xFF); + if (first == ' ' || first == '-') { + // It's an old line. Does it end in CR-LF? + if (buf[nextLineStart - 2] == '\r') { + return true; + } + } + } + lineStart = nextLineStart; + } + } + return false; + } + + private InputStream filterClean(Repository repository, String path, + InputStream fromFile, boolean convertCrLf, String filterCommand) + throws IOException { + InputStream input = fromFile; + if (convertCrLf) { + input = EolStreamTypeUtil.wrapInputStream(input, + EolStreamType.TEXT_LF); + } + if (StringUtils.isEmptyOrNull(filterCommand)) { + return input; + } + if (FilterCommandRegistry.isRegistered(filterCommand)) { + LocalFile buffer = new TemporaryBuffer.LocalFile(null); + FilterCommand command = FilterCommandRegistry.createFilterCommand( + filterCommand, repository, input, buffer); + while (command.run() != -1) { + // loop as long as command.run() tells there is work to do + } + return buffer.openInputStreamWithAutoDestroy(); + } + FS fs = repository.getFS(); + ProcessBuilder filterProcessBuilder = fs.runInShell(filterCommand, + new String[0]); + filterProcessBuilder.directory(repository.getWorkTree()); + filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, + repository.getDirectory().getAbsolutePath()); + ExecutionResult result; + try { + result = fs.execute(filterProcessBuilder, in); + } catch (IOException | InterruptedException e) { + throw new IOException( + new FilterFailedException(e, filterCommand, path)); + } + int rc = result.getRc(); + if (rc != 0) { + throw new IOException(new FilterFailedException(rc, filterCommand, + path, result.getStdout().toByteArray(4096), RawParseUtils + .decode(result.getStderr().toByteArray(4096)))); + } + return result.getStdout().openInputStreamWithAutoDestroy(); + } + /** - * @param f - * @param fh - * @throws IOException - * @throws PatchApplyException + * We write the patch result to a {@link TemporaryBuffer} and then use + * {@link DirCacheCheckout}.getContent() to run the result through the CR-LF + * and smudge filters. DirCacheCheckout needs an ObjectLoader, not a + * TemporaryBuffer, so this class bridges between the two, making the + * TemporaryBuffer look like an ordinary git blob to DirCacheCheckout. */ - private void apply(File f, FileHeader fh) + private static class BufferLoader extends ObjectLoader { + + private TemporaryBuffer data; + + BufferLoader(TemporaryBuffer data) { + this.data = data; + } + + @Override + public int getType() { + return Constants.OBJ_BLOB; + } + + @Override + public long getSize() { + return data.length(); + } + + @Override + public boolean isLarge() { + return true; + } + + @Override + public byte[] getCachedBytes() throws LargeObjectException { + throw new LargeObjectException(); + } + + @Override + public ObjectStream openStream() + throws MissingObjectException, IOException { + return new ObjectStream.Filter(getType(), getSize(), + data.openInputStream()); + } + } + + private void apply(Repository repository, String path, RawText rt, File f, + FileHeader fh, CheckoutMetadata checkOut) throws IOException, PatchApplyException { - RawText rt = new RawText(f); List oldLines = new ArrayList<>(rt.size()); - for (int i = 0; i < rt.size(); i++) + for (int i = 0; i < rt.size(); i++) { oldLines.add(rt.getString(i)); + } List newLines = new ArrayList<>(oldLines); int afterLastHunk = 0; int lineNumberShift = 0; @@ -279,17 +495,32 @@ && canApplyAt(hunkLines, newLines, 0)) { if (!isChanged(oldLines, newLines)) { return; // Don't touch the file } - try (Writer fw = Files.newBufferedWriter(f.toPath())) { - for (Iterator l = newLines.iterator(); l.hasNext();) { - fw.write(l.next()); - if (l.hasNext()) { - // Don't bother handling line endings - if it was Windows, - // the \r is still there! - fw.write('\n'); + + // TODO: forcing UTF-8 is a bit strange and may lead to re-coding if the + // input was some other encoding, but it's what previous versions of + // this code used. (Even earlier the code used the default encoding, + // which has the same problem.) Perhaps using bytes instead of Strings + // for the lines would be better. + TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(null); + try { + try (Writer w = new BufferedWriter( + new OutputStreamWriter(buffer, StandardCharsets.UTF_8))) { + for (Iterator l = newLines.iterator(); l.hasNext();) { + w.write(l.next()); + if (l.hasNext()) { + w.write('\n'); + } } } + try (OutputStream output = new FileOutputStream(f)) { + DirCacheCheckout.getContent(repository, path, checkOut, + new BufferLoader(buffer), null, output); + } + } finally { + buffer.destroy(); } - getRepository().getFS().setExecute(f, fh.getNewMode() == FileMode.EXECUTABLE_FILE); + repository.getFS().setExecute(f, + fh.getNewMode() == FileMode.EXECUTABLE_FILE); } private boolean canApplyAt(List hunkLines, List newLines, From 501fc0dadde1b68a6c7bccd870e18cdf03d0e62c Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Fri, 5 Mar 2021 23:55:18 +0100 Subject: [PATCH 2/8] ApplyCommand: add a base-85 codec Add an implementation for base-85 encoding and decoding [1]. Git binary patches use this format. Base-85 encoding assembles bytes as 32-bit MSB values, then converts these values to base-85 numbers (always 5 bytes) encoded as printable ASCII characters. Decoding base-85 is the reverse operation. Note that decoding may overflow on invalid input as 85^5 > 2^32. Encodings always have a length that is a multiple of 5. If input length is not divisible by 4, padding bytes are (logically) added, which are ignored when decoding. The encoding for n bytes has thus always exactly length (n + 3) / 4 * 5 in integer arithmetic (truncating division). Includes tests. [1] https://datatracker.ietf.org/doc/html/rfc1924 Bug: 371725 Change-Id: Ib5b9a503cd62cf70e080a4fb38c8cd1eeeaebcfe Signed-off-by: Thomas Wolf Signed-off-by: Matthias Sohn --- .../tst/org/eclipse/jgit/util/Base85Test.java | 87 ++++++++ .../eclipse/jgit/internal/JGitText.properties | 5 + .../org/eclipse/jgit/internal/JGitText.java | 5 + .../src/org/eclipse/jgit/util/Base85.java | 195 ++++++++++++++++++ 4 files changed, 292 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/Base85Test.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/Base85.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/Base85Test.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/Base85Test.java new file mode 100644 index 000000000..a49878cc7 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/Base85Test.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; + +import org.junit.Test; + +/** + * Tests for {@link Base85}. + */ +public class Base85Test { + + private static final String VALID_CHARS = "0123456789" + + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + "!#$%&()*+-;<=>?@^_`{|}~"; + + @Test + public void testChars() { + for (int i = 0; i < 256; i++) { + byte[] testData = { '1', '2', '3', '4', (byte) i }; + if (VALID_CHARS.indexOf(i) >= 0) { + byte[] decoded = Base85.decode(testData, 4); + assertNotNull(decoded); + } else { + assertThrows(IllegalArgumentException.class, + () -> Base85.decode(testData, 4)); + } + } + } + + private void roundtrip(byte[] data, int expectedLength) { + byte[] encoded = Base85.encode(data); + assertEquals(expectedLength, encoded.length); + assertArrayEquals(data, Base85.decode(encoded, data.length)); + } + + private void roundtrip(String data, int expectedLength) { + roundtrip(data.getBytes(StandardCharsets.US_ASCII), expectedLength); + } + + @Test + public void testPadding() { + roundtrip("", 0); + roundtrip("a", 5); + roundtrip("ab", 5); + roundtrip("abc", 5); + roundtrip("abcd", 5); + roundtrip("abcde", 10); + roundtrip("abcdef", 10); + roundtrip("abcdefg", 10); + roundtrip("abcdefgh", 10); + roundtrip("abcdefghi", 15); + } + + @Test + public void testBinary() { + roundtrip(new byte[] { 1 }, 5); + roundtrip(new byte[] { 1, 2 }, 5); + roundtrip(new byte[] { 1, 2, 3 }, 5); + roundtrip(new byte[] { 1, 2, 3, 4 }, 5); + roundtrip(new byte[] { 1, 2, 3, 4, 5 }, 10); + roundtrip(new byte[] { 1, 2, 3, 4, 5, 0, 0, 0 }, 10); + roundtrip(new byte[] { 1, 2, 3, 4, 0, 0, 0, 5 }, 10); + } + + @Test + public void testOverflow() { + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> Base85.decode(new byte[] { '~', '~', '~', '~', '~' }, 4)); + assertTrue(e.getMessage().contains("overflow")); + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 2fa8713da..6c4ca52cc 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -37,6 +37,11 @@ badRef=Bad ref: {0}: {1} badSectionEntry=Bad section entry: {0} badShallowLine=Bad shallow line: {0} bareRepositoryNoWorkdirAndIndex=Bare Repository has neither a working tree, nor an index +base85invalidChar=Invalid base-85 character: 0x{0} +base85length=Base-85 encoded data must have a length that is a multiple of 5 +base85overflow=Base-85 value overflow, does not fit into 32 bits: 0x{0} +base85tooLong=Extra base-85 encoded data for output size of {0} bytes +base85tooShort=Base-85 data decoded into less than {0} bytes baseLengthIncorrect=base length incorrect bitmapMissingObject=Bitmap at {0} is missing {1}. bitmapsMustBePrepared=Bitmaps must be prepared before they may be written. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index ab9fc5c9b..5c194f341 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -65,6 +65,11 @@ public static JGitText get() { /***/ public String badSectionEntry; /***/ public String badShallowLine; /***/ public String bareRepositoryNoWorkdirAndIndex; + /***/ public String base85invalidChar; + /***/ public String base85length; + /***/ public String base85overflow; + /***/ public String base85tooLong; + /***/ public String base85tooShort; /***/ public String baseLengthIncorrect; /***/ public String bitmapMissingObject; /***/ public String bitmapsMustBePrepared; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Base85.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Base85.java new file mode 100644 index 000000000..54b7cfcaa --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Base85.java @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Arrays; + +import org.eclipse.jgit.internal.JGitText; + +/** + * Base-85 encoder/decoder. + * + * @since 5.12 + */ +public final class Base85 { + + private static final byte[] ENCODE = ("0123456789" //$NON-NLS-1$ + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" //$NON-NLS-1$ + + "abcdefghijklmnopqrstuvwxyz" //$NON-NLS-1$ + + "!#$%&()*+-;<=>?@^_`{|}~") //$NON-NLS-1$ + .getBytes(StandardCharsets.US_ASCII); + + private static final int[] DECODE = new int[256]; + + static { + Arrays.fill(DECODE, -1); + for (int i = 0; i < ENCODE.length; i++) { + DECODE[ENCODE[i]] = i; + } + } + + private Base85() { + // No instantiation + } + + /** + * Determines the length of the base-85 encoding for {@code rawLength} + * bytes. + * + * @param rawLength + * number of bytes to encode + * @return number of bytes needed for the base-85 encoding of + * {@code rawLength} bytes + */ + public static int encodedLength(int rawLength) { + return (rawLength + 3) / 4 * 5; + } + + /** + * Encodes the given {@code data} in Base-85. + * + * @param data + * to encode + * @return encoded data + */ + public static byte[] encode(byte[] data) { + return encode(data, 0, data.length); + } + + /** + * Encodes {@code length} bytes of {@code data} in Base-85, beginning at the + * {@code start} index. + * + * @param data + * to encode + * @param start + * index of the first byte to encode + * @param length + * number of bytes to encode + * @return encoded data + */ + public static byte[] encode(byte[] data, int start, int length) { + byte[] result = new byte[encodedLength(length)]; + int end = start + length; + int in = start; + int out = 0; + while (in < end) { + // Accumulate remaining bytes MSB first as a 32bit value + long accumulator = ((long) (data[in++] & 0xFF)) << 24; + if (in < end) { + accumulator |= (data[in++] & 0xFF) << 16; + if (in < end) { + accumulator |= (data[in++] & 0xFF) << 8; + if (in < end) { + accumulator |= (data[in++] & 0xFF); + } + } + } + // Write the 32bit value in base-85 encoding, also MSB first + for (int i = 4; i >= 0; i--) { + result[out + i] = ENCODE[(int) (accumulator % 85)]; + accumulator /= 85; + } + out += 5; + } + return result; + } + + /** + * Decodes the Base-85 {@code encoded} data into a byte array of + * {@code expectedSize} bytes. + * + * @param encoded + * Base-85 encoded data + * @param expectedSize + * of the result + * @return the decoded bytes + * @throws IllegalArgumentException + * if expectedSize doesn't match, the encoded data has a length + * that is not a multiple of 5, or there are invalid characters + * in the encoded data + */ + public static byte[] decode(byte[] encoded, int expectedSize) { + return decode(encoded, 0, encoded.length, expectedSize); + } + + /** + * Decodes {@code length} bytes of Base-85 {@code encoded} data, beginning + * at the {@code start} index, into a byte array of {@code expectedSize} + * bytes. + * + * @param encoded + * Base-85 encoded data + * @param start + * index at which the data to decode starts in {@code encoded} + * @param length + * of the Base-85 encoded data + * @param expectedSize + * of the result + * @return the decoded bytes + * @throws IllegalArgumentException + * if expectedSize doesn't match, {@code length} is not a + * multiple of 5, or there are invalid characters in the encoded + * data + */ + public static byte[] decode(byte[] encoded, int start, int length, + int expectedSize) { + if (length % 5 != 0) { + throw new IllegalArgumentException(JGitText.get().base85length); + } + byte[] result = new byte[expectedSize]; + int end = start + length; + int in = start; + int out = 0; + while (in < end && out < expectedSize) { + // Accumulate 5 bytes, "MSB" first + long accumulator = 0; + for (int i = 4; i >= 0; i--) { + int val = DECODE[encoded[in++] & 0xFF]; + if (val < 0) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().base85invalidChar, + Integer.toHexString(encoded[in - 1] & 0xFF))); + } + accumulator = accumulator * 85 + val; + } + if (accumulator > 0xFFFF_FFFFL) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().base85overflow, + Long.toHexString(accumulator))); + } + // Write remaining bytes, MSB first + result[out++] = (byte) (accumulator >>> 24); + if (out < expectedSize) { + result[out++] = (byte) (accumulator >>> 16); + if (out < expectedSize) { + result[out++] = (byte) (accumulator >>> 8); + if (out < expectedSize) { + result[out++] = (byte) accumulator; + } + } + } + } + // Should have exhausted 'in' and filled 'out' completely + if (in < end) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().base85tooLong, + Integer.valueOf(expectedSize))); + } + if (out < expectedSize) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().base85tooShort, + Integer.valueOf(expectedSize))); + } + return result; + } +} From 2eb54afe6a5cb5dd2a108285ad1676b7798d5a15 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 6 Mar 2021 00:00:15 +0100 Subject: [PATCH 3/8] ApplyCommand: add streams to read/write binary patch hunks Add streams that can encode or decode git binary patch data on the fly. Git writes binary patches base-85 encoded, at most 52 un-encoded bytes, with the unencoded data length prefixed in a one-character encoding, and suffixed with a newline character. Add a test for both the new input and the output stream. The test roundtrips binary data of different lengths in different ways. Bug: 371725 Change-Id: Ic3faebaa4637520f5448b3d1acd78d5aaab3907a Signed-off-by: Thomas Wolf --- .../jgit/util/io/BinaryHunkStreamTest.java | 146 ++++++++++++++++++ .../eclipse/jgit/internal/JGitText.properties | 4 + .../org/eclipse/jgit/internal/JGitText.java | 4 + .../jgit/util/io/BinaryHunkInputStream.java | 113 ++++++++++++++ .../jgit/util/io/BinaryHunkOutputStream.java | 116 ++++++++++++++ 5 files changed, 383 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryHunkStreamTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkInputStream.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkOutputStream.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryHunkStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryHunkStreamTest.java new file mode 100644 index 000000000..b198c32a7 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryHunkStreamTest.java @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util.io; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; + +import org.junit.Test; + +/** + * Tests for {@link BinaryHunkInputStream} and {@link BinaryHunkOutputStream}. + */ +public class BinaryHunkStreamTest { + + @Test + public void testRoundtripWholeBuffer() throws IOException { + for (int length = 1; length < 520 + 52; length++) { + byte[] data = new byte[length]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (255 - (i % 256)); + } + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + BinaryHunkOutputStream out = new BinaryHunkOutputStream( + bos)) { + out.write(data); + out.flush(); + byte[] encoded = bos.toByteArray(); + assertFalse(Arrays.equals(data, encoded)); + try (BinaryHunkInputStream in = new BinaryHunkInputStream( + new ByteArrayInputStream(encoded))) { + byte[] decoded = new byte[data.length]; + int newLength = in.read(decoded); + assertEquals(newLength, decoded.length); + assertEquals(-1, in.read()); + assertArrayEquals(data, decoded); + } + } + } + } + + @Test + public void testRoundtripChunks() throws IOException { + for (int length = 1; length < 520 + 52; length++) { + byte[] data = new byte[length]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (255 - (i % 256)); + } + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + BinaryHunkOutputStream out = new BinaryHunkOutputStream( + bos)) { + out.write(data, 0, data.length / 2); + out.write(data, data.length / 2, data.length - data.length / 2); + out.flush(); + byte[] encoded = bos.toByteArray(); + assertFalse(Arrays.equals(data, encoded)); + try (BinaryHunkInputStream in = new BinaryHunkInputStream( + new ByteArrayInputStream(encoded))) { + byte[] decoded = new byte[data.length]; + int p = 0; + int n; + while ((n = in.read(decoded, p, + Math.min(decoded.length - p, 57))) >= 0) { + p += n; + if (p == decoded.length) { + break; + } + } + assertEquals(p, decoded.length); + assertEquals(-1, in.read()); + assertArrayEquals(data, decoded); + } + } + } + } + + @Test + public void testRoundtripBytes() throws IOException { + for (int length = 1; length < 520 + 52; length++) { + byte[] data = new byte[length]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (255 - (i % 256)); + } + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + BinaryHunkOutputStream out = new BinaryHunkOutputStream( + bos)) { + for (int i = 0; i < data.length; i++) { + out.write(data[i]); + } + out.flush(); + byte[] encoded = bos.toByteArray(); + assertFalse(Arrays.equals(data, encoded)); + try (BinaryHunkInputStream in = new BinaryHunkInputStream( + new ByteArrayInputStream(encoded))) { + byte[] decoded = new byte[data.length]; + for (int i = 0; i < decoded.length; i++) { + int val = in.read(); + assertTrue(0 <= val && val <= 255); + decoded[i] = (byte) val; + } + assertEquals(-1, in.read()); + assertArrayEquals(data, decoded); + } + } + } + } + + @Test + public void testRoundtripWithClose() throws IOException { + for (int length = 1; length < 520 + 52; length++) { + byte[] data = new byte[length]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (255 - (i % 256)); + } + try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + try (BinaryHunkOutputStream out = new BinaryHunkOutputStream( + bos)) { + out.write(data); + } + byte[] encoded = bos.toByteArray(); + assertFalse(Arrays.equals(data, encoded)); + try (BinaryHunkInputStream in = new BinaryHunkInputStream( + new ByteArrayInputStream(encoded))) { + byte[] decoded = new byte[data.length]; + int newLength = in.read(decoded); + assertEquals(newLength, decoded.length); + assertEquals(-1, in.read()); + assertArrayEquals(data, decoded); + } + } + } + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 6c4ca52cc..f8c9ea0dc 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -43,6 +43,10 @@ base85overflow=Base-85 value overflow, does not fit into 32 bits: 0x{0} base85tooLong=Extra base-85 encoded data for output size of {0} bytes base85tooShort=Base-85 data decoded into less than {0} bytes baseLengthIncorrect=base length incorrect +binaryHunkDecodeError=Binary hunk, line {0}: invalid input +binaryHunkInvalidLength=Binary hunk, line {0}: input corrupt; expected length byte, got 0x{1} +binaryHunkLineTooShort=Binary hunk, line {0}: input ended prematurely +binaryHunkMissingNewline=Binary hunk, line {0}: input line not terminated by newline bitmapMissingObject=Bitmap at {0} is missing {1}. bitmapsMustBePrepared=Bitmaps must be prepared before they may be written. blameNotCommittedYet=Not Committed Yet diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 5c194f341..85b40cb6d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -71,6 +71,10 @@ public static JGitText get() { /***/ public String base85tooLong; /***/ public String base85tooShort; /***/ public String baseLengthIncorrect; + /***/ public String binaryHunkDecodeError; + /***/ public String binaryHunkInvalidLength; + /***/ public String binaryHunkLineTooShort; + /***/ public String binaryHunkMissingNewline; /***/ public String bitmapMissingObject; /***/ public String bitmapsMustBePrepared; /***/ public String blameNotCommittedYet; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkInputStream.java new file mode 100644 index 000000000..57b2d7a4b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkInputStream.java @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util.io; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.StreamCorruptedException; +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.util.Base85; + +/** + * A stream that decodes git binary patch data on the fly. + * + * @since 5.12 + */ +public class BinaryHunkInputStream extends InputStream { + + private final InputStream in; + + private int lineNumber; + + private byte[] buffer; + + private int pos = 0; + + /** + * Creates a new {@link BinaryHunkInputStream}. + * + * @param in + * {@link InputStream} to read the base-85 encoded patch data + * from + */ + public BinaryHunkInputStream(InputStream in) { + this.in = in; + } + + @Override + public int read() throws IOException { + if (pos < 0) { + return -1; + } + if (buffer == null || pos == buffer.length) { + fillBuffer(); + } + if (pos >= 0) { + return buffer[pos++] & 0xFF; + } + return -1; + } + + @Override + public void close() throws IOException { + in.close(); + buffer = null; + } + + private void fillBuffer() throws IOException { + int length = in.read(); + if (length < 0) { + pos = length; + buffer = null; + return; + } + lineNumber++; + // Length is encoded with characters, A..Z for 1..26 and a..z for 27..52 + if ('A' <= length && length <= 'Z') { + length = length - 'A' + 1; + } else if ('a' <= length && length <= 'z') { + length = length - 'a' + 27; + } else { + throw new StreamCorruptedException(MessageFormat.format( + JGitText.get().binaryHunkInvalidLength, + Integer.valueOf(lineNumber), Integer.toHexString(length))); + } + byte[] encoded = new byte[Base85.encodedLength(length)]; + for (int i = 0; i < encoded.length; i++) { + int b = in.read(); + if (b < 0 || b == '\n') { + throw new EOFException(MessageFormat.format( + JGitText.get().binaryHunkInvalidLength, + Integer.valueOf(lineNumber))); + } + encoded[i] = (byte) b; + } + // Must be followed by a newline; tolerate EOF. + int b = in.read(); + if (b >= 0 && b != '\n') { + throw new StreamCorruptedException(MessageFormat.format( + JGitText.get().binaryHunkMissingNewline, + Integer.valueOf(lineNumber))); + } + try { + buffer = Base85.decode(encoded, length); + } catch (IllegalArgumentException e) { + StreamCorruptedException ex = new StreamCorruptedException( + MessageFormat.format(JGitText.get().binaryHunkDecodeError, + Integer.valueOf(lineNumber))); + ex.initCause(e); + throw ex; + } + pos = 0; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkOutputStream.java new file mode 100644 index 000000000..30551c09f --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryHunkOutputStream.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util.io; + +import java.io.IOException; +import java.io.OutputStream; + +import org.eclipse.jgit.util.Base85; + +/** + * An {@link OutputStream} that encodes data for a git binary patch. + * + * @since 5.12 + */ +public class BinaryHunkOutputStream extends OutputStream { + + private static final int MAX_BYTES = 52; + + private final OutputStream out; + + private final byte[] buffer = new byte[MAX_BYTES]; + + private int pos; + + /** + * Creates a new {@link BinaryHunkOutputStream}. + * + * @param out + * {@link OutputStream} to write the encoded data to + */ + public BinaryHunkOutputStream(OutputStream out) { + this.out = out; + } + + /** + * Flushes and closes this stream, and closes the underlying + * {@link OutputStream}. + */ + @Override + public void close() throws IOException { + flush(); + out.close(); + } + + /** + * Writes any buffered output as a binary patch line to the underlying + * {@link OutputStream} and flushes that stream, too. + */ + @Override + public void flush() throws IOException { + if (pos > 0) { + encode(buffer, 0, pos); + pos = 0; + } + out.flush(); + } + + @Override + public void write(int b) throws IOException { + buffer[pos++] = (byte) b; + if (pos == buffer.length) { + encode(buffer, 0, pos); + pos = 0; + } + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (len == 0) { + return; + } + int toCopy = len; + int in = off; + if (pos > 0) { + // Fill the buffer + int chunk = Math.min(toCopy, buffer.length - pos); + System.arraycopy(b, in, buffer, pos, chunk); + in += chunk; + pos += chunk; + toCopy -= chunk; + if (pos == buffer.length) { + encode(buffer, 0, pos); + pos = 0; + } + if (toCopy == 0) { + return; + } + } + while (toCopy >= MAX_BYTES) { + encode(b, in, MAX_BYTES); + toCopy -= MAX_BYTES; + in += MAX_BYTES; + } + if (toCopy > 0) { + System.arraycopy(b, in, buffer, 0, toCopy); + pos = toCopy; + } + } + + private void encode(byte[] data, int off, int length) throws IOException { + if (length <= 26) { + out.write('A' + length - 1); + } else { + out.write('a' + length - 27); + } + out.write(Base85.encode(data, off, length)); + out.write('\n'); + } +} From 0fe794a433d54504de066ae119b5835ab69c1c54 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sun, 7 Mar 2021 18:49:01 +0100 Subject: [PATCH 4/8] ApplyCommand: add a stream to apply a delta patch Add a new BinaryDeltaInputStream that applies a delta provided by another InputStream to a given base. Because delta application needs random access to the base, the base itself cannot be yet another InputStream. But at least this enables streaming of the result. Add a simple test using delta hunks generated by C git. Bug: 371725 Change-Id: Ibd26fa2f49860737ad5c5387f7f4870d3e85e628 Signed-off-by: Thomas Wolf Signed-off-by: Matthias Sohn --- .../org/eclipse/jgit/util/io/delta1.forward | Bin 0 -> 27 bytes .../org/eclipse/jgit/util/io/delta1.reverse | Bin 0 -> 27 bytes .../util/io/BinaryDeltaInputStreamTest.java | 103 +++++++++ .../eclipse/jgit/internal/JGitText.properties | 3 + .../org/eclipse/jgit/internal/JGitText.java | 3 + .../jgit/util/io/BinaryDeltaInputStream.java | 206 ++++++++++++++++++ 6 files changed, 315 insertions(+) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/util/io/delta1.forward create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/util/io/delta1.reverse create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryDeltaInputStreamTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryDeltaInputStream.java diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/util/io/delta1.forward b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/util/io/delta1.forward new file mode 100644 index 0000000000000000000000000000000000000000..878b167ae90209a601dcd6eea47abefccd5d26e9 GIT binary patch literal 27 icmWGe&W$QCh{!EZG_2CnRtYuEicc=~({RbH;sOAE?Fg{| literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/util/io/delta1.reverse b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/util/io/delta1.reverse new file mode 100644 index 0000000000000000000000000000000000000000..7ff7a08ad0a12a8be94888f4dd92e0bf78cf0d1d GIT binary patch literal 27 icmWGZ&W$QCjmUK|tu9rpNVGN0aLqKbuy+eE;Q|1EYzR>R literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryDeltaInputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryDeltaInputStreamTest.java new file mode 100644 index 000000000..d9297fcd9 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/BinaryDeltaInputStreamTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util.io; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.zip.InflaterInputStream; + +import org.junit.Test; + +/** + * Crude tests for the {@link BinaryDeltaInputStream} using delta diffs + * generated by C git. + */ +public class BinaryDeltaInputStreamTest { + + private InputStream getBinaryHunk(String name) { + return this.getClass().getResourceAsStream(name); + } + + @Test + public void testBinaryDelta() throws Exception { + // Prepare our test data + byte[] data = new byte[8192]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) (255 - (i % 256)); + } + // Same, but with five 'x' inserted in the middle. + int middle = data.length / 2; + byte[] newData = new byte[data.length + 5]; + System.arraycopy(data, 0, newData, 0, middle); + for (int i = 0; i < 5; i++) { + newData[middle + i] = 'x'; + } + System.arraycopy(data, middle, newData, middle + 5, middle); + // delta1.forward has the instructions + // @formatter:off + // COPY 0 4096 + // INSERT 5 xxxxx + // COPY 0 4096 + // @formatter:on + // Note that the way we built newData could be expressed as + // @formatter:off + // COPY 0 4096 + // INSERT 5 xxxxx + // COPY 4096 4096 + // @formatter:on + try (ByteArrayOutputStream out = new ByteArrayOutputStream(); + BinaryDeltaInputStream input = new BinaryDeltaInputStream(data, + new InflaterInputStream(new BinaryHunkInputStream( + getBinaryHunk("delta1.forward"))))) { + byte[] buf = new byte[1024]; + int n; + while ((n = input.read(buf)) >= 0) { + out.write(buf, 0, n); + } + assertArrayEquals(newData, out.toByteArray()); + assertTrue(input.isFullyConsumed()); + } + // delta1.reverse has the instructions + // @formatter:off + // COPY 0 4096 + // COPY 256 3840 + // COPY 256 256 + // @formatter:on + // Note that there are alternatives, for instance + // @formatter:off + // COPY 0 4096 + // COPY 4101 4096 + // @formatter:on + // or + // @formatter:off + // COPY 0 4096 + // COPY 0 4096 + // @formatter:on + try (ByteArrayOutputStream out = new ByteArrayOutputStream(); + BinaryDeltaInputStream input = new BinaryDeltaInputStream( + newData, + new InflaterInputStream(new BinaryHunkInputStream( + getBinaryHunk("delta1.reverse"))))) { + long expectedSize = input.getExpectedResultSize(); + assertEquals(data.length, expectedSize); + byte[] buf = new byte[1024]; + int n; + while ((n = input.read(buf)) >= 0) { + out.write(buf, 0, n); + } + assertArrayEquals(data, out.toByteArray()); + assertTrue(input.isFullyConsumed()); + } + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index f8c9ea0dc..8b6872d33 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -43,6 +43,9 @@ base85overflow=Base-85 value overflow, does not fit into 32 bits: 0x{0} base85tooLong=Extra base-85 encoded data for output size of {0} bytes base85tooShort=Base-85 data decoded into less than {0} bytes baseLengthIncorrect=base length incorrect +binaryDeltaBaseLengthMismatch=Binary delta base length does not match, expected {0}, got {1} +binaryDeltaInvalidOffset=Binary delta offset + length too large: {0} + {1} +binaryDeltaInvalidResultLength=Binary delta expected result length is negative binaryHunkDecodeError=Binary hunk, line {0}: invalid input binaryHunkInvalidLength=Binary hunk, line {0}: input corrupt; expected length byte, got 0x{1} binaryHunkLineTooShort=Binary hunk, line {0}: input ended prematurely diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 85b40cb6d..3b67b0f62 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -71,6 +71,9 @@ public static JGitText get() { /***/ public String base85tooLong; /***/ public String base85tooShort; /***/ public String baseLengthIncorrect; + /***/ public String binaryDeltaBaseLengthMismatch; + /***/ public String binaryDeltaInvalidOffset; + /***/ public String binaryDeltaInvalidResultLength; /***/ public String binaryHunkDecodeError; /***/ public String binaryHunkInvalidLength; /***/ public String binaryHunkLineTooShort; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryDeltaInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryDeltaInputStream.java new file mode 100644 index 000000000..7f0d87f0d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/BinaryDeltaInputStream.java @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util.io; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.StreamCorruptedException; +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; + +/** + * An {@link InputStream} that applies a binary delta to a base on the fly. + *

+ * Delta application to a base needs random access to the base data. The delta + * is expressed as a sequence of copy and insert instructions. A copy + * instruction has the form "COPY fromOffset length" and says "copy length bytes + * from the base, starting at offset fromOffset, to the result". An insert + * instruction has the form "INSERT length" followed by length bytes and says + * "copy the next length bytes from the delta to the result". + *

+ *

+ * These instructions are generated using a content-defined chunking algorithm + * (currently C git uses the standard Rabin variant; but there are others that + * could be used) that identifies equal chunks. It is entirely possible that a + * later copy instruction has a fromOffset that is before the fromOffset of an + * earlier copy instruction. + *

+ *

+ * This makes it impossible to stream the base. + *

+ *

+ * JGit is limited to 2GB maximum size for the base since array indices are + * signed 32bit values. + * + * @since 5.12 + */ +public class BinaryDeltaInputStream extends InputStream { + + private final byte[] base; + + private final InputStream delta; + + private long resultLength; + + private long toDeliver = -1; + + private int fromBase; + + private int fromDelta; + + private int baseOffset = -1; + + /** + * Creates a new {@link BinaryDeltaInputStream} that applies {@code delta} + * to {@code base}. + * + * @param base + * data to apply the delta to + * @param delta + * {@link InputStream} delivering the delta to apply + */ + public BinaryDeltaInputStream(byte[] base, InputStream delta) { + this.base = base; + this.delta = delta; + } + + @Override + public int read() throws IOException { + int b = readNext(); + if (b >= 0) { + toDeliver--; + } + return b; + } + + private void initialize() throws IOException { + long baseSize = readVarInt(delta); + if (baseSize > Integer.MAX_VALUE || baseSize < 0 + || (int) baseSize != base.length) { + throw new IOException(MessageFormat.format( + JGitText.get().binaryDeltaBaseLengthMismatch, + Integer.valueOf(base.length), Long.valueOf(baseSize))); + } + resultLength = readVarInt(delta); + if (resultLength < 0) { + throw new StreamCorruptedException( + JGitText.get().binaryDeltaInvalidResultLength); + } + toDeliver = resultLength; + baseOffset = 0; + } + + private int readNext() throws IOException { + if (baseOffset < 0) { + initialize(); + } + if (fromBase > 0) { + fromBase--; + return base[baseOffset++] & 0xFF; + } else if (fromDelta > 0) { + fromDelta--; + return delta.read(); + } + int command = delta.read(); + if (command < 0) { + return -1; + } + if ((command & 0x80) != 0) { + // Decode offset and length to read from base + long copyOffset = 0; + for (int i = 1, shift = 0; i < 0x10; i *= 2, shift += 8) { + if ((command & i) != 0) { + copyOffset |= ((long) next(delta)) << shift; + } + } + int copySize = 0; + for (int i = 0x10, shift = 0; i < 0x80; i *= 2, shift += 8) { + if ((command & i) != 0) { + copySize |= next(delta) << shift; + } + } + if (copySize == 0) { + copySize = 0x10000; + } + if (copyOffset > base.length - copySize) { + throw new StreamCorruptedException(MessageFormat.format( + JGitText.get().binaryDeltaInvalidOffset, + Long.valueOf(copyOffset), Integer.valueOf(copySize))); + } + baseOffset = (int) copyOffset; + fromBase = copySize; + return readNext(); + } else if (command != 0) { + // The next 'command' bytes come from the delta + fromDelta = command - 1; + return delta.read(); + } else { + // Zero is reserved + throw new StreamCorruptedException( + JGitText.get().unsupportedCommand0); + } + } + + private int next(InputStream in) throws IOException { + int b = in.read(); + if (b < 0) { + throw new EOFException(); + } + return b; + } + + private long readVarInt(InputStream in) throws IOException { + long val = 0; + int shift = 0; + int b; + do { + b = next(in); + val |= ((long) (b & 0x7f)) << shift; + shift += 7; + } while ((b & 0x80) != 0); + return val; + } + + /** + * Tells the expected size of the final result. + * + * @return the size + * @throws IOException + * if the size cannot be determined from {@code delta} + */ + public long getExpectedResultSize() throws IOException { + if (baseOffset < 0) { + initialize(); + } + return resultLength; + } + + /** + * Tells whether the delta has been fully consumed, and the expected number + * of bytes for the combined result have been read from this + * {@link BinaryDeltaInputStream}. + * + * @return whether delta application was successful + */ + public boolean isFullyConsumed() { + try { + return toDeliver == 0 && delta.read() < 0; + } catch (IOException e) { + return toDeliver == 0; + } + } + + @Override + public void close() throws IOException { + delta.close(); + } +} From 10ac4499115965ff10e547a0632c89873a06cf91 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sun, 7 Mar 2021 18:50:23 +0100 Subject: [PATCH 5/8] ApplyCommand: support binary patches Implement applying binary patches. Handles both literal and delta patches. Note that C git also runs binary files through the clean and smudge filters. Implement the same safeguards against corrupted patches as in C git: require the full OIDs to be present in the patch file, and apply a binary patch only if both pre- and post-image hashes match. Add tests for applying literal and delta patches. Bug: 371725 Change-Id: I71dc214fe4145d7cc8e4769384fb78c7d0d6c220 Signed-off-by: Thomas Wolf --- .../org/eclipse/jgit/diff/.gitattributes | Bin 39 -> 67 bytes .../org/eclipse/jgit/diff/delta.patch | Bin 0 -> 213 bytes .../org/eclipse/jgit/diff/delta_PostImage | Bin 0 -> 8197 bytes .../org/eclipse/jgit/diff/delta_PreImage | Bin 0 -> 8192 bytes .../org/eclipse/jgit/diff/literal.patch | Bin 0 -> 8090 bytes .../org/eclipse/jgit/diff/literal_PostImage | Bin 0 -> 5389 bytes .../org/eclipse/jgit/diff/literal_PreImage | Bin 0 -> 1629 bytes .../org/eclipse/jgit/diff/literal_add.patch | Bin 0 -> 2316 bytes .../eclipse/jgit/diff/literal_add_PostImage | Bin 0 -> 1629 bytes .../eclipse/jgit/api/ApplyCommandTest.java | 41 +++ .../eclipse/jgit/internal/JGitText.properties | 3 + .../org/eclipse/jgit/api/ApplyCommand.java | 274 ++++++++++++++++-- .../org/eclipse/jgit/internal/JGitText.java | 3 + 13 files changed, 291 insertions(+), 30 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_add.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_add_PostImage diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/.gitattributes b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/.gitattributes index c5831d9a8b46a6685b6c952d367bea807e452153..28caa2f82a1af374b3a1ca4cd42faf36cb93f197 100644 GIT binary patch delta 33 icmY#)o}ez1lA2SJsHLD=l3G#1m6KVLT9lXr;{pJ)*$Tk` delta 4 LcmZ=(pP&u^0?Yv- diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta.patch new file mode 100644 index 0000000000000000000000000000000000000000..84855308a5b5b846b1d98df579c4fccdd8c533d0 GIT binary patch literal 213 zcmXxd%}PTt5J2I5o+3gQqO@k_W^&U->84uT6u}mX8=1)@ZKSspz0x0FUZl7>n-7lG zQtWWJslDX~&AXnhTx;rH^T~=QY@H)1f6g7;?rytGpH{CTzq?$g#>_u$QI>fR1+-5nV&clU!;^YNa( g7PqtRV*U}D02Ln-foH`&cnfpI*HA1!?C#=l}o! literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta_PostImage new file mode 100644 index 0000000000000000000000000000000000000000..8b370bb5f2bc3261b6b62e80d6edd784a61ec225 GIT binary patch literal 8197 zcmezW@9&@AzkdGs{_X3R&!0Yic>nJ0o7b;izIguZ>66Eg9zM8#@9v%3w{G6Je(ma& z%a<-*IDhW!nbW6Eo;ZH&=#j&R4j$OQZ||PnyLRr_zHRH4&6_rESif%Vn$@dTu2{Zo z>5|2Z7A}}SZ|F(<6Xm4w6X>Mw4sIRN7sjjN5C@(85 zDK083$j{5o$mNlr>kh>weniH?el2oDPl2@VPj@b~le@%HlcaCdWcadvWa zu(z|dv9_|bFgG(bF*Y(Z(AU$|(bm$`P*+n`QC3n^ke8E{k(QE_5El~_5f%~@;OFDz z;pXDxU}s}xVP;}v_&@6Z(fA)t|D*YTwEP<_|3~Y;(fWV1{WIGB8*TrNw*N=_KcoG> z5zzmw0D;j?@aQ1O=pfMOAlT?2;OHRe=pgXuAo%Dc!005%=p@kSB-rRA;OHdi=p^v) Hp9BX0J?`^5 literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/delta_PreImage new file mode 100644 index 0000000000000000000000000000000000000000..b4527005bf9e4da2dd1d7185b51bdac623a4218f GIT binary patch literal 8192 zcmezW@9&@AzkdGs{_X3R&!0Yic>nJ0o7b;izIguZ>66Eg9zM8#@9v%3w{G6Je(ma& z%a<-*IDhW!nbW6Eo;ZH&=#j&R4j$OQZ||PnyLRr_zHRH4&6_rESif%Vn$@dTu2{Zo z>5|2Z7A}}SZ|F(<6Xm4w6X>Mw4sIRN7sjjN5C@(85 zDK083$j{5o$mNlr>kh>weniH?el2oDPl2@VPj@b~le@%HlcaCdWcadvWa zu(z|dv9_|bFgG(bF*Y(Z(AU$|(bm$`P*+n`QC3n^ke8E{k(QE_5El~_5f%~@;OFDz z;pXDxU}s}xVP;}v_&@6Z(fA)t|D*YTwEP<_|3~Y;(fWV1{WIGB8*TrNw*N=_KcoG> z(f;pf|9^D+V|4sybo^^{{BLypb9DT7bo_gC{C{-*V|4y!bpC5}{%>^tb9DZ97|eeI E0NPLR;s5{u literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal.patch new file mode 100644 index 0000000000000000000000000000000000000000..c8811d5bb0ec32adca7c0eb1626c664e4856cb2a GIT binary patch literal 8090 zcmX|`*Up06x`ofbtN4v*RN~r7H)2Bt5mdV60Hj(F!GcoMYhU~aYsbVGOvo(ne8(8i zbWWc4PqTSAJ^wNP?{a#&ck}X(^}k<#r>o;8|KOdD;}IxMxUg-ygo(NxywfHixYM=> zw_{=^?6}3}KmWlI7lB%B)5Bn#Fl_?2ypG#$6P{&S7>dF+?l~}u+W)j$Eet{a7*_em zI$h0s`cGs&?c?A7hJeCE=P$DZyIS3JJqS&pUh6=cwL=&1R>T~1zk z@U|M>-cs0rO&Zn(_g345n|zD>Wo+K;4TPElA`;_{cOxYh;Z}Pj)-`VYtj9v*9Q4c9 zG24InroLSs*p=iTX*!*-4>^>3+udJw1**uoHOcZ@a4U)FxQBiuk-5Keyv=^4(TA)U zDN!8=Cx}p8Lh?NZP$r*Pm*7t$QEM6Xm+gieDpX-GanIw0t=g?^i3JAhGu8+1Vq(^* z?`M@AUTbYnMZITl$b<3qWgp#blV;Z1j6VHk7x1W#nC?^L8D@3gPFZVbyc42+5z2#< zd3;5}1d!@}^qibuue10L^0a5<)s@99>e)N?wZCjIibm7$)vGk35p}O2C2Z9NJr4ar zcs(6AzL~)hA1GRG=;>}%qo1e56RnA+lim}HYlrjqFO$a~0%~M}F2gLTv}3jYSdORb z5nnCUzuX%DQ*)#f1lLzo%IzghRxR(A-5n6`xlwC+ypH}dJ}hC?W)H0zJM3$`;96&P ze<81Fo=LSXu8<{oDN#G4oMy6pzO_2d^08rWt{_`0F+=s28Gl)6cNhsaxxZ&ZW!Nwp z@@9JJBzniA=E7uxWTn;&itnTcjjZP6f7Vv7pzMeC&My&=c6_9pdK*!$=U@LlT;A0RAg2xpi|fA7ES=^-SL z;Zy@t-v=1HT#rUjjmyj$okYvM*VaCb>xZ+Q6Efb~z7$9@096wtnXf^Z0@3@B=7G&&uMpI#22iG;db;%JEO5II|) zPfQ0V{V#)CJ5@LH?l6COoq^+`b@ku?SwAR6PTPNetJh^w(iLF*LZ57T&i|f^lDBaJ znD`yy1BOI@nIO)xM{=JE;p}JHaeNYP?}N+xo6h5T8QPh%zX&H?Wrg_0)-H-rdpcZZ z&aa1N`-e}Pr{`a$e`%zyrao{Ps`wn<;8t7Ng{LfBT+9zy=TAbZz4Xh-tn4wYl$WxX zdN45ij{6>Vb{q`*%O)k!F4H{+;!pBy#SviAWn1Ig_S~b85<{?E%)!ISPOs?#WtlSD zbJWyK*z~)7r#hbP*}qH*`7rG1Yrp>CD`4lf*|I}&x%QOa6tEQi=Jh|aDv2VqA(l5j zTHd?Y4txD2z;@qJ)fln!YtW%q2Lb_SM?CZfVvACwX) zX15jubC9cm0iMo)RhAWbb#5Ym}0W2r@rxR zo3-|h4@?#_3jz$sIy&L}91kQ9prFMZ_nu)W|7CcvC#f#`b!rOG>$Mu(l5IZq z&K~MzV*PHu_nAA&cc0ndUVoN13_qBwo&?d#bwwhO6G=>Q*s0k`xl~8E9F`L3qWM~~ za)#MyK6eskTonMfFHVdd{3^vI*6X4oOONcz6kI~*ew*eFMP1;P-FbE*Z>r5i&EMN? z<8Yc)D@}9u$ebIF?f2VZA3}nY_()A$TG!sT|EO?o^yp@B-D(w5^nlR0aHLlQ$fRG> zsimD=s9r5yDQ3xHFLS>;7vfhXyNz(fTCdiplG#eIeIr$56*toWJ zo7=q#=t*->-5oc_re@rR0zD#!rumTBhJo^0>Kt&CwDGmIk_PS0bIO4GqUBI+NHCtQ z&c5la+D?A3*43qYAqZmlNuSM2wbboKsi7}p(xJHb#D>$?Wb*64KXFFULbmzK0H*wr z>JaxDCBt+(I%%-KwXIXE8p}%j5*O0+a@wyAbzqz2R_dGaq9He%4|?x-YeYrg;$Jq! zV+($Mj`QE$9E~ucaoppqe-rOfjS4`|e$=)aaOwA#QQ3C}=icb}9H08-B$*ED$UOKI z?U&q2Rc~VOp$3wda%|`#x8J9$~Y`L`wx9$q_+dUKS&Fkc;5y!OD+1R&5qtDs- z>{Q#xwe*;{ryS1T>0w>Uthj?xYzuDKGE;4+vd{X}uAfuzuO zEpvj01Qs3qDb?6l4CGVVZ3JSf-8^0mO&<0JdlDU$Riw4GF8gY2YvMg?<driDHS8Br#i=HU2vj2{M6dkRvpJn;US%!4j zx;sUFoX8|Usd!;gFP5pv= ze)g}u*nYVUKW?<#N#!^F%ecp~=lUCKkLAup&&nEB$2egB)dvZ%nydVpiBhXFtj@J% z9ewl52EHxR;g~S`?-o=K*4@ACb$I*2btl;F`COZC(9-0$^ncp<{_d1NjaJI&dV4(| z4oPS)eXPiufHtXqriM!ZNqrV^J#fij!Sf*L;7mgh}$9mw)+s_ zZFsz{2E#*M(*j|4$|6NDOo`iCh*Ah*<8pc29M~>|7KZ(Kmw^IZx|82iZa~7frdZ>Q(F{UX>6EdunHM z>D#u#Ve(^Ddm;=bRJ5fcjkSfcsu6aLU3H2bai-p^BJ2ojRjyX1`LvmCFKY`UPwK`G z`aXrJ^7%|`Z_nBU7;Z)|RVw!?#^il_f$!6uA_@bo-wy9loY2{67W>osU=OxhOzmCZ{3M?k%4yJPT`scVrCp=-cYNdc}@N)c!?P8z$F?0mow&jy*2{ z_&qqU=_bE8e+v0Hm)ff+F5ep2;BqRjwI$3@&1>tKDD1{8bfjK5m{Y}UVS^2jKQ#YS z*Y$(moiyL31Ad(qV(KHUNtETwYyZ@c<&+-~Go4^Y9+50c)vH%HzUa7~q4w+y_YX`P z_ttT(9A1y_IYrhUa?;We^^!UF8BNtQJw5wFirddnr&zd zfV<(P$MM3?J?yQm)NYhDjpzKpmWROM*RaR%@nfQWvc_?p)?#yf+-uF^q&6lSuM;a` zt-GlHc;~I;Ev?(oklB9BJ?98~TGl*tE^qm@bita9XVd7M#0u8FsH1twR|_?(@!*_^ zqSCLn{Z1q9wA&A@UUaX$dSPFFs-ynrQ&TN&dn6-CU$kBszn7iSsBX$PKe7+`~pZPj^gqd@n2&ej%>X zD~!^4TL}M|1%T2?ez7vqSPwDrH=yOyxD}d9DiNCta+!wpTO}O~=R};5oVVL)WL8s> zTz`y<*nc0Y(BiOBDr{<)yT(bn+!h<89}5t=e-WrBz)1Ph%~n!sd?M-j3B&5lmiJs9 zem2_)R0Qrhp1G8X*TchuGs8x&K0k!3wL?pdFOIj4Y?Nz>sUId>&GiiGT9ce=2O4=g z*Z(99YY^Rv4(BAN?$K|+(`R^?O(kWIZU#kwLf~Bny_o7g+m44B)9I59F8uJx z_AMEdzY~_gneCT+E5udWQ>y~RNXZa2%r-Mc2i!`4ECCFalx=drkRvMb0;SUuI#h%~ zw^2%z2ny)mY$Z9{5(M^HtIGoq^g4J&_N5HSvg%Q*HG>eyc#_pN@49KXt;I+cC= zUP`CERlnxQ&)SGj-@KcEvh$W%5Dh(_z=Muaw)paP;ZLmN z_c$;>C1zT3Z!n&Y-|fy)aHjq2s(v1u4*4MpH7lF@gqX*JphB0IMVZo<``)g8VK9E4 z=ndassVh}6Af`RIlBa=7)whI&UXD4TV=JoqDHU7`Kg{dE3k6$2UfS)`EJp<3a#{M! zxj+pBJ-5f-*N0zV!;7TnG;=a{&|dzzZ-Xme7Gv^T80hI0R<>pNP`kNZdx{jWt4g}AD7@=_(do+cA~)^CUGbTz1k z$;s~<8Y*f~HfxP=pc(}e^u=s~)1X54U%I|7UcBl_P!k}|{lfir%8v9hPX>ncm@l&z z^TVVZDAoo0S@0~VpDz}JmL3Cc&cIcPF@VqZ?VQC^C!dz(&-7hzE&m|F4zQECMGpyk z%XIbJk00rmsq}^qOjxHjmP8n4jrTf!rrC8U!luvIhpg9`%jej82ZeRrcfX_2dAPx{ z+2nF}gKta$jclV6m(GV8d|)lwx&!GfA4|!p1C+3jzDaKZNt4Y*uLW$}f++a`^h>5? zHwe=@6iC~qP#0y+ zy%a+}0G}nLtzBxMzdz4K|1~2nmsZx$`9Rfux3ehq8IW4kmCk}pPa}KZ?sa{$u<^TB z4PHHIH+u~>cUz~oj#V0uRe6nT=R{Q+_v2tX${(}kLY{Wi=jJvi8oD0F&3^TMyq8`MX2HlCrjt`+p4!cA zA906(j{MolAzmW>^;BD`Fo{+2F5I5aqgn$<^q9GKJjUF;x?6no_y6R&jX{OYvjeSp zhA6=}33c+=g}D#!w^}W?y`1yPt<`E%>wmx8wqDei)hW|N@ox*U;IjRNuG?f?^cm7x z%Ed!gT>2MsC);u=wuRJ&Us|M&?UPAYOaH{Y+u1?TF|g_x}+2^cPU ztDv>-?(F-jGa~C@75GfdeihH`o}K;?z&fZXt!;k|>T(UhrpR@(?ruT;F+2QD8Vyw& z`vrHD{<2vRjK#Kve{E}0LVRu%RZmP@;dy5q z&942AveXLVKuEA@e3UE*$_-c;ip`m$R#~4gAZNGRM0-_o~``v;9raBuOF9$C9q_v3z?RNzn(^=yuJb?OJ@- zSKq@7sZi`}NOZfsiLv4fzj{c8(lF4AL@|K%+r&ErmuAQ7>H6d{1xt#XMdLm|6_kh= zi0<|{Bir*EEwp;x;2Z3IuUetSNjQgZdusV`T&pqeRl)442!~q8?E1=M88jlS`Oryc zT8~9l&)re!O%U{>Qy-;oX=?PFzrQRbGm_Y@P%((zYDM7XL(+X|#t}c`Dj%`bY4brG zv{I7mpD`&eX0LJ|y`2`X)82-?&G}jcvcWZD`&YPiJEUAKRf6{T44y(d*WqDnyuQn; zdjEdS?QcG0LO1sK%>#Vx7^S9Wu%q?~)4t%G+RWK`w7(7+ z_PbM(W=J8x_H9SN>1G!9U>}vKj6G&eKIRucPQ+Yd<-+*=OO)I^Y77i3%!o!*^g{2-k3%mkONb!MHt?i7pXTs8YSMaT@JmC z0v0;HJ_f#9D!QX$EL8ecX!;LMrQ3)7kGXBq){I)h=ZLQFo>n^7_WF3 zkGxlEUO#-jurV3v^ZO^q<%k%N^_f1Er016^l5tY@LIF61U$HBUTH21P6|sB2GRi= zebJy+1!8x2=vn-nljw`G#GA$4Q39yzOK2a)$2vT^{Kg%r{_}S9uTk80BHF(OD;!l- z{|yg05SRG_9-NOi<+lDFHE_=#GaI$Ap3kc|i@rK*vRak>+KfdGt6_rRt!~&UOvM(w zE9pGFdL{!_bv{BnBXWU`>jgI3TF3Im8&2qN(M zLGd?vgzcb1>+x)Zi6Gs7p6EBPY@Y}8@`#+z9^H0TDJ{goV??*1nGyNYeWtpsFz#NL zJHK#>qsvNO_DNJQ`CbX@3i6wZSC8k>>JDE|a|zFgl6PoIZ(I})Y+vvPNXPsfvW33N zl&8As6$MV_Pf#u^r_HU7ecWmDkjZT8E*Ri8*iuOPMcMx3F-ZCJ6^wGk34;xGmC}!5 z<@WIKhMBMk>Z7zTR>s{H)Oe(mY-z55u4-z-2#qzNnIzbGt=0g7jkYyvlC4DijNP#bhz`}L-^YNgIORkhFFyDIsSoS=YZ zn$&4j0Dz{qmq!QyC=5|RQGrv{71I}RA|<%_y8&?Pi28Ue8RkfG$TD|u^R3|*xU(zB zZ@K5P&3+xejCMcUc0Y!I$?Wj6>+rQxa)B}|OzgvWV`Bp=t=@L+unvcheY-FGWVHFi zl+hUAG;tQLGGOrYEIbZ_wx8)lP#?yGs}Q8a$4&`AtDjvvjQwm{y=_{&ObKM)zwukV z?ApAEov=%*pABr(4!6O;+fK5~27)yC*}z_K8|2pNXFEZO(FtixfQbDV^TUBe=x-IU#ZN;Z$u`KA0EhnA zLo!6&-=5&_Z6}_v3(9Hrv4vE{zR(;7T997ReQrr7yVJYLD z`dVs3lYV+~Opo@m+<1RMi}Zelv(aO{zFtaOw^J2so#)_}Rh^3mTy5Ulm5u0bd1$sd z%1SaNcGG8UC;ePqht{0v`kKJ63u}K_vov5v_U_lm!{_Bh8qW{?ZT<4o4jb||uFmi^ z30ZU^a{;@{vf9?de5)Ugd$z>m!pd1kt}&nNxfQ%$Cv5tI#JL~tM!fXdm38~gug64p zPQHzsxrEPoH~acuuiPqwgeLkX9stl@czd`lPi^UIPuqI(lTP(tdKzaJng-1~7E-it z^}F4)Uyk$l@U7J3X7b1TZoizoMla|<)LiP!1Nk#4H#fGIR{!zvUg_6cr7kf6jdd66 zD!Qc#F=d2LiSHOK8+x;I0C(Yj=B4!G8I;6d7oDZ^(^PBbaSI=dUyaxIy}Ywy^7z`I zRoLoo=^Wk)|L%+9vXYI@|SURQQGX8NqLpNr;n9In~x$Y`9UUu4b z=jT}O9W^IA>!s^fWqQta2@~XR4QTc%_W6`nbn@Ne3#VgO$O3|`e3XWjkL%mQV8j1RJ?y=7L@4SY`!CkuNp2*<-VN6L3(>ruD2jFae>PqZX-`bsIQ&+JMt{gP zMGkCfgv$FQnqcs~f~OdiY|DPg!rmxM$kF4{{`Zqm(~$&i z+QAv2st76{Qlo6S65)=WRxC)yUy z;w?RL0hdV)Qm>* zBES}EqfdtubT$WSPJnZ!yI~I$dFb&`ckB_IEzK!M8+oC^5DKk{7mRf$B2MJ7ftooS z7m`QD_Z3v!u!If;5QIFAc%E+N2j>1FKgg z7%ZjHi24miQsya7RW(wcks%Keps;zu)b0A zs25JPhFO?2aFvEEbVje2)&!nBGNI6va`ZjV^<&<;*oAIbv0|%zcTmYw4&4OpddEAL z6i|AB&Whx=JaY`U$^x-?&ZhXh-V_q@PFA<@aasSI*_5LVydg75>Np#kd?j=AhtCi>#xxE@)4Jm|^ zs9C!Q5#be(DKW;PJqrzQr-nDtV8hy!9XcqM*>Kc{Hi)Y zPmTmm2c(`po-3dPH8V+oB5Go4z_^Jy6~>KBLl`$QO+h7U>07{50ljGbu$HFq9!STl zRQU5*u}TW)qryW}P=qcYIYJU%0T=Mg2!9T1opK-OQxml?Ex>xTe&mRTFc(zdmsR+t ztR(Ivu${EFe*mQyVqfLUoaL3f?`<5A#U40Ybl`&(Ya7=bOd)MOJ2G-U=Z>H$_T|0Y z^{W@KQn>cOguJ0?aGWf-7(JT(@|I5iBf%`}C~BFpWn|N#tKb_RgYeZTSsiw02)mW; zvgwkPnZPcejj(JX5!D~Nk~Vjrw9#|*0yG^^1WJ@@<2`R|zz2lx1OkxK8@F%oTSx+v zp`5$JW&137yLLy#O#u!FXR4^OH*u1oLwc>28fXL7)Um@g{(VmPZizpTP_aRKYb4TI z*llsqd3ccr733kJ;G2m^Rr<}i{_2P}cOcP_#NKNavgOg9e~mw%r_cj!h+n+Q(e7dY zC7;|xx2D1TQPpU?h~b)&6iQnTo2o=U9Xl>ccTK6SsO(t4G6P%VNTRCVMTO(8c(IO9 ztdEfH%PZ8bO#&%E{E-yYlWb6kB6USjj8Yqt(D$@XA3B!#=%_Dwqv+wirB8 zL^qgwAuxFziHukE)?`+00Gm~>VH3$dQxU2SgG17fWu(#onhwJB<(sSx0V4UlzdrHG%y;vzb`345ye5k10>)4L5IP; zI`Paa7l+a12C$+Z@l=KLP~eg`bYR1*#VVLKkmcS@$g~*cC3h|a5><(KK90MXhsr(- zS*DXf9uTQ(;M&~LdwT{0V;*V4H`CKtJ43iN(f5JSYLWRl6-*t-_9iQ%61lUAJ!kUg z(L+JGpp!~(Y}EpKiZ`@b=pr(v=o>StUA>||(UbKYhJ5 z-VuxUVj8GE-%=*eT>9*iKn*b4#VW5Z`H%uP25`v85fP34#l;zORy_|pRt|oo&HcS`K{o|XYD2B zJ7zeBa7)ogsKao*=)JgoOWdFR*}770hIp{Vxa^gDK(ef_Bn(Rd&92Af^K>|x*b?pq z^dPeNYe@*U035~{RBN)ZOyG{>0&8ibxsV2O6<*3EtogHERT`5X?k$L43Zz2vG&`zYC@zG9><*Ko|Ln#1{*GL0XV_zF;!)3W>if zpd#f+`~!h1Qi#Mq5}?RAB(=wijm3dgvIBbgde~LPJMuNMyC(Rxh+U4g zUUkq6#}v28CoPlCukTBh>Mh`0>y4h}Tg&ESk7r!yE=O>(dSt(ALywPCZytwK9m#d$ zUPt$*@ckG@_#W!myqe{smjppbnDdfPTRpkQP{j*IeS42pi12jTuZ6ys1T5r_(hW9$ zdU7+-&x42VYT#sQ({uaa88Y>$n@=VcSoT;2uznU|sX9<|8I z#a`x8Ju?m%F#NNYeFlr`c&}3+utiFXL*IBnB%_2G1?n1oIEO?&oxik>_V5QV2zH0A7$&pmD5vGXmi?%F>v^^WO~RO=R2-$8j> z$9#dRvai_64=G}G`Q{gD#mTbl6~b3Hm)jlbG5!s?KJG3&RG_$|YwdjQi9{E#S|Ew} zkgR3d$ZSKM=4o=NyOt`Csy}G%=@s6PWyc6xp66bC`^7jFIWoQ3akGcgz=9JSjufwm zP`;}AAj@X6zHDlrem}uB$*zX^8l9|B?|3VpYe?RjSh!xi9$!6|^XyqmvH8ll0z*^LmU^ke9M!>G+BM)T9M zvbft3*&THeF9!a=qS6Gt{Qt`(rJo+ZM=+B3-uVBe{r`m2>TuR2Gvx|d^>(M0_1^{J z&OOgkw(AystYHt|E^z)78usIKcv{%8i?zQQxwP+SKCpk(P++5}ym5b3NZWww$fh${ z+z&czyqYIeTzcz0oSZKBoa(Z#PTn*0gDZ_->&rhr!vd6b34w3Fxi*Jyc-Y>4e@$^4 cxguN4kni(NU3<6%{;31JJp(*$x^uGr9fGaZv;Y7A literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_PreImage new file mode 100644 index 0000000000000000000000000000000000000000..799df8578e3cae8a5e979182391b8e9a6a3deded GIT binary patch literal 1629 zcmV-j2BP_iP)XOp-ouEG7tw0 zrz1-=F%F0OAZ*Bt%4AWe%cd6G%m`sY6u$yfhB9DOrfkevkPjQ|7bz{Y^t;geeV+Z% z-b-(9d+9KXPx9t{&pFR|p5ODF^FHT&;Y${|kx3oD%S#{awv*l7K)tMQzWRR$@NLb_ zw;H}v)GQIf6%3j8H~%fa;jd>h$695QRWckW+^S($6E4nXW;Y+LsHg~iVF2&*`MKhR zHmmCQ1byyU`lD^F&Xm|TNV z%8R3a_|Tzmm#*3P@Q-HSUYM1eH!c3jMs;&&F z8VCRQ^d#Seu9v{M5A2zz4}Ep>na1}yj<$BEqf8?QeYhH<1JqFP0=f?}mZ~v(g zxoJ3q`BNsdeb*C=&+znTPg(oGZ>TsNm*p6-s^-~ zo5qiMvi`c3fXjnY z?nzI>n30qgyK(yTm1Cj+ioF{^3@0CWrSO)>rn{c}{@GoeufLXs+vZGT(^CeYib z@v#)Z=<7IOmIA1i6V}+JBXJIJp`n@GI}ZTx-oB4$Ywcp$iqa857|UG%CfGW821q7= zq9+APM+z`d>S`Lf|K3fsw_Ssnls$oQl&LdL&c>{lV@vUe( zB{i9~>+T+MNs|1@lU%4xMhqz7rXF{kl4D#XI1 zbX{p6Z2g0st1+xPRr~fQk-nI>L$^#A<5PaL=!6;PSwBuOsthb;@vtaN3H8!fg-Rw7QQgI>Pk?49a|gzM^OzTtUX=( z<{t+S#TiXq)6|eE-gzSI0rP_+YmYB4y}5I>Ds?yLyvmA{4domX6nRy|Tb{c@gw3Go zSAy>Q%C;02*syDzNaO$;UaV@p5uVP-xx6jWU($8IpY(BDzOQ7j6qV%)(*@c8YURW; zzpcp2S4#n^S%{1S+QBwkHC1k7_I_Hk`;+V09uYtc%=Ww#?-e@}G}|!}PU;OD{-Qsp bU%LDkMDBWX&Ru`<00000NkvXXu0mjfJA@b~ literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_add.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_add.patch new file mode 100644 index 0000000000000000000000000000000000000000..bb6a9e42a33680860c068af59e1feb687c72567f GIT binary patch literal 2316 zcma)8+0LqJ620dsexn$buvot$*~va6Cxs!|s# zE2{=)?|DBc>hEtq=6{a+n|n2nThnoVtpClRll%Db_DA<8Iy>$UR;?2CIEY6!kRUl!mOrCnS_Zu?yo~bQU9^_C-eIHiOriGK>q^! z!H60K0XtfthZ-wy_i~vDXng#Z!e0=FpciSQkiEkve&%2;pP7$)a1dJb=!WZ^T`rx#K0zowtb!mr7WTTY|#Grm!EolRh62brO z7ksT)*-gfQ3IQRC`U+;fUi00N+wjFz#Khwyz)cl5w{IOpQNDu9T?G}ixiP_9A;(zxoPfIP#rip950ZCY^|4}s>QuPPfSuem98&nAst4S@P*5VucE8hAv!jS=h))~8*g(f39t z0|}0K9a9%gtAlwmum7T$T(S?QO7|6*-uy$F=ZM~&bPQ$H85k$~Rn56vD;V8`>7Y=U z>&o}EG|N-*IrwBmt6HQN2n43uSaPNDy=b>h?DIM>3|qe?Qy!L5?UDqH5*C&XV<|SU zQhMFYKE6w%6yZ#z1*YO_wvB!vaApA0G-eGYO5GwXOPc?r?~83f9>}>Ai|l^& zt~(7lrCtYK2L7;ojm)9+Z6lamx!}i4DgN)eKNF2qx zQ+FtS*7_4Yv-N`Uu;ar1V*W0N_k0NVz2UN1!!BeC0v8JD$rA+ZoVK=;y!ROQlE+p{ zPRFEhRv6)$4$?L2+LOezJLgE;`d0G&qO8OLyzJS9-S3#W*yD!Ad3|x60DkW0^X8Z~ zy^oTQAG!0@md)a-K8T%ocue)E0($;pz zjY}^QYr*}!CD%*6=M+|8yRN*)^l|{d;;^r@JTXNoFq0n#WH6C)5NKVSIJLg9wxccz z;%sLz2HAuUCBz$qRxrO0VRRrL&;g#B^4eZsNqZ2RrF~hJHp?5|2uTQx?QDx9#Oslg zut!Ms$84UJ{f~F9aGlNQ`p!r#Zg!Zmuo-9rIPu_bdO}i@yc*f4N+6%##mByyYMj)B z_h$4yAT{KB?brEJCorm`zPyzJa{rc)In}9{N2GWQd4836RiUpIlJPBr(P;$&xT+Ee zABpj&5tWLC)b08$OB#(;{-{q$w7Ah;@I~eb&h;eWoG^ z7|2zBjb10u93sVy6gQ|IGimh8EBj(tLH_7m=M<*iZciP;Uyul5` zHUQO`}3PSWGCny`&S^(xF=QSQ^H z={oE|pm&^U(>BwPO{tY5Xjyx(=`6Vg7Tvwy%qcE3bDoc?4^mNlu`P$4I%U3nFxkw2 zew(jazXE5n8=~d){pwOC-tGYPfhJb3@)j{7gRQ?l4YAE7otN~wN4>1>oTC+SQeSV~ z%IvQHz=vlDXuVfkf%|l|R!LqQn3}T!JvSHKxREYo^!X$X%Xz8L6l0^ra$u&A2ja4T z%cV-!ZW?oSNkag{c3jiy`O`*&GfYDi_7MtwEF`z$RtR@AvZzw2akJsp@{#YLZuc|% zymKFUPS3auRgxI8=CH(t6OLW3ifX!J{$Bp-G`z1?X*D`{qh8ngLjV0Is*v#eOmv*T HO&IhKS0dI+ literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_add_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/literal_add_PostImage new file mode 100644 index 0000000000000000000000000000000000000000..799df8578e3cae8a5e979182391b8e9a6a3deded GIT binary patch literal 1629 zcmV-j2BP_iP)XOp-ouEG7tw0 zrz1-=F%F0OAZ*Bt%4AWe%cd6G%m`sY6u$yfhB9DOrfkevkPjQ|7bz{Y^t;geeV+Z% z-b-(9d+9KXPx9t{&pFR|p5ODF^FHT&;Y${|kx3oD%S#{awv*l7K)tMQzWRR$@NLb_ zw;H}v)GQIf6%3j8H~%fa;jd>h$695QRWckW+^S($6E4nXW;Y+LsHg~iVF2&*`MKhR zHmmCQ1byyU`lD^F&Xm|TNV z%8R3a_|Tzmm#*3P@Q-HSUYM1eH!c3jMs;&&F z8VCRQ^d#Seu9v{M5A2zz4}Ep>na1}yj<$BEqf8?QeYhH<1JqFP0=f?}mZ~v(g zxoJ3q`BNsdeb*C=&+znTPg(oGZ>TsNm*p6-s^-~ zo5qiMvi`c3fXjnY z?nzI>n30qgyK(yTm1Cj+ioF{^3@0CWrSO)>rn{c}{@GoeufLXs+vZGT(^CeYib z@v#)Z=<7IOmIA1i6V}+JBXJIJp`n@GI}ZTx-oB4$Ywcp$iqa857|UG%CfGW821q7= zq9+APM+z`d>S`Lf|K3fsw_Ssnls$oQl&LdL&c>{lV@vUe( zB{i9~>+T+MNs|1@lU%4xMhqz7rXF{kl4D#XI1 zbX{p6Z2g0st1+xPRr~fQk-nI>L$^#A<5PaL=!6;PSwBuOsthb;@vtaN3H8!fg-Rw7QQgI>Pk?49a|gzM^OzTtUX=( z<{t+S#TiXq)6|eE-gzSI0rP_+YmYB4y}5I>Ds?yLyvmA{4domX6nRy|Tb{c@gw3Go zSAy>Q%C;02*syDzNaO$;UaV@p5uVP-xx6jWU($8IpY(BDzOQ7j6qV%)(*@c8YURW; zzpcp2S4#n^S%{1S+QBwkHC1k7_I_Hk`;+V09uYtc%=Ww#?-e@}G}|!}PU;OD{-Qsp bU%LDkMDBWX&Ru`<00000NkvXXu0mjfJA@b~ literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java index 335a64d70..e9b8924e6 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java @@ -9,6 +9,7 @@ */ package org.eclipse.jgit.api; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -19,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import org.eclipse.jgit.api.errors.PatchApplyException; import org.eclipse.jgit.api.errors.PatchFormatException; @@ -29,6 +31,7 @@ import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.util.IO; import org.junit.Test; public class ApplyCommandTest extends RepositoryTestCase { @@ -246,6 +249,44 @@ public void testFiltering() throws Exception { } } + private void checkBinary(String name, boolean hasPreImage) + throws Exception { + try (Git git = new Git(db)) { + byte[] post = IO + .readWholeStream(getTestResource(name + "_PostImage"), 0) + .array(); + File f = new File(db.getWorkTree(), name); + if (hasPreImage) { + byte[] pre = IO + .readWholeStream(getTestResource(name + "_PreImage"), 0) + .array(); + Files.write(f.toPath(), pre); + git.add().addFilepattern(name).call(); + git.commit().setMessage("PreImage").call(); + } + ApplyResult result = git.apply() + .setPatch(getTestResource(name + ".patch")).call(); + assertEquals(1, result.getUpdatedFiles().size()); + assertEquals(f, result.getUpdatedFiles().get(0)); + assertArrayEquals(post, Files.readAllBytes(f.toPath())); + } + } + + @Test + public void testBinaryDelta() throws Exception { + checkBinary("delta", true); + } + + @Test + public void testBinaryLiteral() throws Exception { + checkBinary("literal", true); + } + + @Test + public void testBinaryLiteralAdd() throws Exception { + checkBinary("literal_add", false); + } + @Test public void testAddA1() throws Exception { ApplyResult result = init("A1", false, true); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 8b6872d33..962324e0f 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -13,6 +13,9 @@ ambiguousObjectAbbreviation=Object abbreviation {0} is ambiguous aNewObjectIdIsRequired=A NewObjectId is required. anExceptionOccurredWhileTryingToAddTheIdOfHEAD=An exception occurred while trying to add the Id of HEAD anSSHSessionHasBeenAlreadyCreated=An SSH session has been already created +applyBinaryBaseOidWrong=Cannot apply binary patch; OID for file {0} does not match +applyBinaryOidTooShort=Binary patch for file {0} does not have full IDs +applyBinaryResultOidWrong=Result of binary patch for file {0} has wrong OID. applyingCommit=Applying {0} archiveFormatAlreadyAbsent=Archive format already absent: {0} archiveFormatAlreadyRegistered=Archive format already registered with different implementation: {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java index 5d975ea38..f7eba88ac 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java @@ -9,7 +9,9 @@ */ package org.eclipse.jgit.api; +import java.io.BufferedInputStream; import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -25,6 +27,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.zip.InflaterInputStream; import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.api.errors.GitAPIException; @@ -44,10 +47,13 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectStream; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.patch.BinaryHunk; import org.eclipse.jgit.patch.FileHeader; +import org.eclipse.jgit.patch.FileHeader.PatchType; import org.eclipse.jgit.patch.HunkHeader; import org.eclipse.jgit.patch.Patch; import org.eclipse.jgit.treewalk.FileTreeIterator; @@ -57,14 +63,17 @@ import org.eclipse.jgit.treewalk.filter.NotIgnoredFilter; import org.eclipse.jgit.treewalk.filter.PathFilterGroup; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.ExecutionResult; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.TemporaryBuffer; -import org.eclipse.jgit.util.FS.ExecutionResult; import org.eclipse.jgit.util.TemporaryBuffer.LocalFile; +import org.eclipse.jgit.util.io.BinaryDeltaInputStream; +import org.eclipse.jgit.util.io.BinaryHunkInputStream; import org.eclipse.jgit.util.io.EolStreamTypeUtil; +import org.eclipse.jgit.util.sha1.SHA1; /** * Apply a patch to files and/or to the index. @@ -191,6 +200,9 @@ private File getFile(String path, boolean create) private void apply(Repository repository, String path, DirCache cache, File f, FileHeader fh) throws IOException, PatchApplyException { + if (PatchType.BINARY.equals(fh.getPatchType())) { + return; + } boolean convertCrLf = needsCrLfConversion(f, fh); // Use a TreeWalk with a DirCacheIterator to pick up the correct // clean/smudge filters. CR-LF handling is completely determined by @@ -217,16 +229,23 @@ private void apply(Repository repository, String path, DirCache cache, FileTreeIterator file = walk.getTree(fileIdx, FileTreeIterator.class); if (file != null) { - command = walk - .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN); - RawText raw; - // Can't use file.openEntryStream() as it would do CR-LF - // conversion as usual, not as wanted by us. - try (InputStream input = filterClean(repository, path, - new FileInputStream(f), convertCrLf, command)) { - raw = new RawText(IO.readWholeStream(input, 0).array()); + if (PatchType.GIT_BINARY.equals(fh.getPatchType())) { + applyBinary(repository, path, f, fh, + file::openEntryStream, file.getEntryObjectId(), + checkOut); + } else { + command = walk.getFilterCommand( + Constants.ATTR_FILTER_TYPE_CLEAN); + RawText raw; + // Can't use file.openEntryStream() as it would do CR-LF + // conversion as usual, not as wanted by us. + try (InputStream input = filterClean(repository, path, + new FileInputStream(f), convertCrLf, command)) { + raw = new RawText( + IO.readWholeStream(input, 0).array()); + } + applyText(repository, path, raw, f, fh, checkOut); } - apply(repository, path, raw, f, fh, checkOut); return; } } @@ -234,21 +253,30 @@ private void apply(Repository repository, String path, DirCache cache, // File ignored? RawText raw; CheckoutMetadata checkOut; - if (convertCrLf) { - try (InputStream input = EolStreamTypeUtil.wrapInputStream( - new FileInputStream(f), EolStreamType.TEXT_LF)) { - raw = new RawText(IO.readWholeStream(input, 0).array()); - } - checkOut = new CheckoutMetadata(EolStreamType.TEXT_CRLF, null); - } else { - raw = new RawText(f); + if (PatchType.GIT_BINARY.equals(fh.getPatchType())) { checkOut = new CheckoutMetadata(EolStreamType.DIRECT, null); + applyBinary(repository, path, f, fh, () -> new FileInputStream(f), + null, checkOut); + } else { + if (convertCrLf) { + try (InputStream input = EolStreamTypeUtil.wrapInputStream( + new FileInputStream(f), EolStreamType.TEXT_LF)) { + raw = new RawText(IO.readWholeStream(input, 0).array()); + } + checkOut = new CheckoutMetadata(EolStreamType.TEXT_CRLF, null); + } else { + raw = new RawText(f); + checkOut = new CheckoutMetadata(EolStreamType.DIRECT, null); + } + applyText(repository, path, raw, f, fh, checkOut); } - apply(repository, path, raw, f, fh, checkOut); } private boolean needsCrLfConversion(File f, FileHeader fileHeader) throws IOException { + if (PatchType.GIT_BINARY.equals(fileHeader.getPatchType())) { + return false; + } if (!hasCrLf(fileHeader)) { try (InputStream input = new FileInputStream(f)) { return RawText.isCrLfText(input); @@ -258,7 +286,7 @@ private boolean needsCrLfConversion(File f, FileHeader fileHeader) } private static boolean hasCrLf(FileHeader fileHeader) { - if (fileHeader == null) { + if (PatchType.GIT_BINARY.equals(fileHeader.getPatchType())) { return false; } for (HunkHeader header : fileHeader.getHunks()) { @@ -330,19 +358,30 @@ private InputStream filterClean(Repository repository, String path, return result.getStdout().openInputStreamWithAutoDestroy(); } + /** + * Something that can supply an {@link InputStream}. + */ + private interface StreamSupplier { + InputStream load() throws IOException; + } + /** * We write the patch result to a {@link TemporaryBuffer} and then use * {@link DirCacheCheckout}.getContent() to run the result through the CR-LF * and smudge filters. DirCacheCheckout needs an ObjectLoader, not a - * TemporaryBuffer, so this class bridges between the two, making the - * TemporaryBuffer look like an ordinary git blob to DirCacheCheckout. + * TemporaryBuffer, so this class bridges between the two, making any Stream + * provided by a {@link StreamSupplier} look like an ordinary git blob to + * DirCacheCheckout. */ - private static class BufferLoader extends ObjectLoader { + private static class StreamLoader extends ObjectLoader { - private TemporaryBuffer data; + private StreamSupplier data; - BufferLoader(TemporaryBuffer data) { + private long size; + + StreamLoader(StreamSupplier data, long length) { this.data = data; + this.size = length; } @Override @@ -352,7 +391,7 @@ public int getType() { @Override public long getSize() { - return data.length(); + return size; } @Override @@ -369,12 +408,146 @@ public byte[] getCachedBytes() throws LargeObjectException { public ObjectStream openStream() throws MissingObjectException, IOException { return new ObjectStream.Filter(getType(), getSize(), - data.openInputStream()); + new BufferedInputStream(data.load())); } } - private void apply(Repository repository, String path, RawText rt, File f, - FileHeader fh, CheckoutMetadata checkOut) + private void initHash(SHA1 hash, long size) { + hash.update(Constants.encodedTypeString(Constants.OBJ_BLOB)); + hash.update((byte) ' '); + hash.update(Constants.encodeASCII(size)); + hash.update((byte) 0); + } + + private ObjectId hash(File f) throws IOException { + SHA1 hash = SHA1.newInstance(); + initHash(hash, f.length()); + try (InputStream input = new FileInputStream(f)) { + byte[] buf = new byte[8192]; + int n; + while ((n = input.read(buf)) >= 0) { + hash.update(buf, 0, n); + } + } + return hash.toObjectId(); + } + + private void checkOid(ObjectId baseId, ObjectId id, ChangeType type, File f, + String path) + throws PatchApplyException, IOException { + boolean hashOk = false; + if (id != null) { + hashOk = baseId.equals(id); + if (!hashOk && ChangeType.ADD.equals(type) + && ObjectId.zeroId().equals(baseId)) { + // We create the file first. The OID of an empty file is not the + // zero id! + hashOk = Constants.EMPTY_BLOB_ID.equals(id); + } + } else { + if (ObjectId.zeroId().equals(baseId)) { + // File empty is OK. + hashOk = !f.exists() || f.length() == 0; + } else { + hashOk = baseId.equals(hash(f)); + } + } + if (!hashOk) { + throw new PatchApplyException(MessageFormat + .format(JGitText.get().applyBinaryBaseOidWrong, path)); + } + } + + private void applyBinary(Repository repository, String path, File f, + FileHeader fh, StreamSupplier loader, ObjectId id, + CheckoutMetadata checkOut) + throws PatchApplyException, IOException { + if (!fh.getOldId().isComplete() || !fh.getNewId().isComplete()) { + throw new PatchApplyException(MessageFormat + .format(JGitText.get().applyBinaryOidTooShort, path)); + } + BinaryHunk hunk = fh.getForwardBinaryHunk(); + // A BinaryHunk has the start at the "literal" or "delta" token. Data + // starts on the next line. + int start = RawParseUtils.nextLF(hunk.getBuffer(), + hunk.getStartOffset()); + int length = hunk.getEndOffset() - start; + SHA1 hash = SHA1.newInstance(); + // Write to a buffer and copy to the file only if everything was fine + TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(null); + try { + switch (hunk.getType()) { + case LITERAL_DEFLATED: + // This just overwrites the file. We need to check the hash of + // the base. + checkOid(fh.getOldId().toObjectId(), id, fh.getChangeType(), f, + path); + initHash(hash, hunk.getSize()); + try (OutputStream out = buffer; + InputStream inflated = new SHA1InputStream(hash, + new InflaterInputStream( + new BinaryHunkInputStream( + new ByteArrayInputStream( + hunk.getBuffer(), start, + length))))) { + DirCacheCheckout.getContent(repository, path, checkOut, + new StreamLoader(() -> inflated, hunk.getSize()), + null, out); + if (!fh.getNewId().toObjectId().equals(hash.toObjectId())) { + throw new PatchApplyException(MessageFormat.format( + JGitText.get().applyBinaryResultOidWrong, + path)); + } + } + try (InputStream bufIn = buffer.openInputStream()) { + Files.copy(bufIn, f.toPath(), + StandardCopyOption.REPLACE_EXISTING); + } + break; + case DELTA_DEFLATED: + // Unfortunately delta application needs random access to the + // base to construct the result. + byte[] base; + try (InputStream input = loader.load()) { + base = IO.readWholeStream(input, 0).array(); + } + // At least stream the result! + try (BinaryDeltaInputStream input = new BinaryDeltaInputStream( + base, + new InflaterInputStream(new BinaryHunkInputStream( + new ByteArrayInputStream(hunk.getBuffer(), + start, length))))) { + long finalSize = input.getExpectedResultSize(); + initHash(hash, finalSize); + try (OutputStream out = buffer; + SHA1InputStream hashed = new SHA1InputStream(hash, + input)) { + DirCacheCheckout.getContent(repository, path, checkOut, + new StreamLoader(() -> hashed, finalSize), null, + out); + if (!fh.getNewId().toObjectId() + .equals(hash.toObjectId())) { + throw new PatchApplyException(MessageFormat.format( + JGitText.get().applyBinaryResultOidWrong, + path)); + } + } + } + try (InputStream bufIn = buffer.openInputStream()) { + Files.copy(bufIn, f.toPath(), + StandardCopyOption.REPLACE_EXISTING); + } + break; + default: + break; + } + } finally { + buffer.destroy(); + } + } + + private void applyText(Repository repository, String path, RawText rt, + File f, FileHeader fh, CheckoutMetadata checkOut) throws IOException, PatchApplyException { List oldLines = new ArrayList<>(rt.size()); for (int i = 0; i < rt.size(); i++) { @@ -514,7 +687,9 @@ && canApplyAt(hunkLines, newLines, 0)) { } try (OutputStream output = new FileOutputStream(f)) { DirCacheCheckout.getContent(repository, path, checkOut, - new BufferLoader(buffer), null, output); + new StreamLoader(buffer::openInputStream, + buffer.length()), + null, output); } } finally { buffer.destroy(); @@ -565,4 +740,43 @@ private boolean isNoNewlineAtEndOfFile(FileHeader fh) { return lhrt.getString(lhrt.size() - 1) .equals("\\ No newline at end of file"); //$NON-NLS-1$ } + + /** + * An {@link InputStream} that updates a {@link SHA1} on every byte read. + * The hash is supposed to have been initialized before reading starts. + */ + private static class SHA1InputStream extends InputStream { + + private final SHA1 hash; + + private final InputStream in; + + SHA1InputStream(SHA1 hash, InputStream in) { + this.hash = hash; + this.in = in; + } + + @Override + public int read() throws IOException { + int b = in.read(); + if (b >= 0) { + hash.update((byte) b); + } + return b; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n = in.read(b, off, len); + if (n > 0) { + hash.update(b, off, n); + } + return n; + } + + @Override + public void close() throws IOException { + in.close(); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 3b67b0f62..fd54986f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -41,6 +41,9 @@ public static JGitText get() { /***/ public String aNewObjectIdIsRequired; /***/ public String anExceptionOccurredWhileTryingToAddTheIdOfHEAD; /***/ public String anSSHSessionHasBeenAlreadyCreated; + /***/ public String applyBinaryBaseOidWrong; + /***/ public String applyBinaryOidTooShort; + /***/ public String applyBinaryResultOidWrong; /***/ public String applyingCommit; /***/ public String archiveFormatAlreadyAbsent; /***/ public String archiveFormatAlreadyRegistered; From 76b76a6048b9b3dbca965463d4f6f5732ffb784d Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Wed, 10 Mar 2021 14:25:37 +0100 Subject: [PATCH 6/8] ApplyCommand: use byte arrays for text patches, not strings Instead of converting the patch bytes to strings apply the patch on byte level, like C git does. Converting the input lines and the hunk lines from bytes to strings and then applying the patch based on strings may give surprising results if a patch converts a text file from one encoding to another. Moreover, in the end we don't know which encoding to use to write the result. Previous code just wrote the result as UTF-8, which forcibly changed the encoding if the original input had some other encoding (even if the patch had the same non-UTF-8 encoding). It was also wrong if the input was UTF-8, and the patch should have changed the encoding to something else. So use ByteBuffers instead of Strings. This has the additional advantage that all these ByteBuffers can share the underlying byte arrays of the input and of the patch, so it also reduces memory consumption. Change-Id: I450975f2ba0e7d0bec8973e3113cc2e7aea187ee Signed-off-by: Thomas Wolf --- .../org.eclipse.core.resources.prefs | 3 +- .../org/eclipse/jgit/diff/umlaut.patch | Bin 0 -> 110 bytes .../org/eclipse/jgit/diff/umlaut_PostImage | Bin 0 -> 4 bytes .../org/eclipse/jgit/diff/umlaut_PreImage | Bin 0 -> 7 bytes .../eclipse/jgit/api/ApplyCommandTest.java | 8 ++ .../org/eclipse/jgit/api/ApplyCommand.java | 70 ++++++++---------- .../src/org/eclipse/jgit/diff/RawText.java | 24 +++++- 7 files changed, 65 insertions(+), 40 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/umlaut.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/umlaut_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/umlaut_PreImage diff --git a/org.eclipse.jgit.test/.settings/org.eclipse.core.resources.prefs b/org.eclipse.jgit.test/.settings/org.eclipse.core.resources.prefs index 6a9621db1..cddb99d1d 100644 --- a/org.eclipse.jgit.test/.settings/org.eclipse.core.resources.prefs +++ b/org.eclipse.jgit.test/.settings/org.eclipse.core.resources.prefs @@ -1,5 +1,6 @@ -#Sat Dec 20 21:21:24 CET 2008 eclipse.preferences.version=1 +encoding//tst-rsrc/org/eclipse/jgit/diff/umlaut.patch=ISO-8859-1 +encoding//tst-rsrc/org/eclipse/jgit/diff/umlaut_PostImage=ISO-8859-1 encoding//tst-rsrc/org/eclipse/jgit/patch/testGetText_BothISO88591.patch=ISO-8859-1 encoding//tst-rsrc/org/eclipse/jgit/patch/testGetText_Convert.patch=ISO-8859-1 encoding//tst-rsrc/org/eclipse/jgit/patch/testGetText_DiffCc.patch=ISO-8859-1 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/umlaut.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/umlaut.patch new file mode 100644 index 0000000000000000000000000000000000000000..7380dbed822eaf9ef3016fb181f19286d0c8bd48 GIT binary patch literal 110 zcmYevOiNSH)lJVVQApG;&CN+HEm25 oldLines = new ArrayList<>(rt.size()); + List oldLines = new ArrayList<>(rt.size()); for (int i = 0; i < rt.size(); i++) { - oldLines.add(rt.getString(i)); + oldLines.add(rt.getRawString(i)); } - List newLines = new ArrayList<>(oldLines); + List newLines = new ArrayList<>(oldLines); int afterLastHunk = 0; int lineNumberShift = 0; int lastHunkNewLine = -1; @@ -571,9 +568,9 @@ private void applyText(Repository repository, String path, RawText rt, b.length); RawText hrt = new RawText(b); - List hunkLines = new ArrayList<>(hrt.size()); + List hunkLines = new ArrayList<>(hrt.size()); for (int i = 0; i < hrt.size(); i++) { - hunkLines.add(hrt.getString(i)); + hunkLines.add(hrt.getRawString(i)); } if (hh.getNewStartLine() == 0) { @@ -642,8 +639,8 @@ && canApplyAt(hunkLines, newLines, 0)) { lineNumberShift = applyAt - hh.getNewStartLine() + 1; int sz = hunkLines.size(); for (int j = 1; j < sz; j++) { - String hunkLine = hunkLines.get(j); - switch (hunkLine.charAt(0)) { + ByteBuffer hunkLine = hunkLines.get(j); + switch (hunkLine.array()[hunkLine.position()]) { case ' ': applyAt++; break; @@ -651,7 +648,7 @@ && canApplyAt(hunkLines, newLines, 0)) { newLines.remove(applyAt); break; case '+': - newLines.add(applyAt++, hunkLine.substring(1)); + newLines.add(applyAt++, slice(hunkLine, 1)); break; default: break; @@ -660,28 +657,29 @@ && canApplyAt(hunkLines, newLines, 0)) { afterLastHunk = applyAt; } if (!isNoNewlineAtEndOfFile(fh)) { - newLines.add(""); //$NON-NLS-1$ + newLines.add(null); } if (!rt.isMissingNewlineAtEnd()) { - oldLines.add(""); //$NON-NLS-1$ + oldLines.add(null); } - if (!isChanged(oldLines, newLines)) { - return; // Don't touch the file + if (oldLines.equals(newLines)) { + return; // Unchanged; don't touch the file } - // TODO: forcing UTF-8 is a bit strange and may lead to re-coding if the - // input was some other encoding, but it's what previous versions of - // this code used. (Even earlier the code used the default encoding, - // which has the same problem.) Perhaps using bytes instead of Strings - // for the lines would be better. TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(null); try { - try (Writer w = new BufferedWriter( - new OutputStreamWriter(buffer, StandardCharsets.UTF_8))) { - for (Iterator l = newLines.iterator(); l.hasNext();) { - w.write(l.next()); + try (OutputStream out = buffer) { + for (Iterator l = newLines.iterator(); l + .hasNext();) { + ByteBuffer line = l.next(); + if (line == null) { + // Must be the marker for the final newline + break; + } + out.write(line.array(), line.position(), + line.limit() - line.position()); if (l.hasNext()) { - w.write('\n'); + out.write('\n'); } } } @@ -698,18 +696,18 @@ && canApplyAt(hunkLines, newLines, 0)) { fh.getNewMode() == FileMode.EXECUTABLE_FILE); } - private boolean canApplyAt(List hunkLines, List newLines, - int line) { + private boolean canApplyAt(List hunkLines, + List newLines, int line) { int sz = hunkLines.size(); int limit = newLines.size(); int pos = line; for (int j = 1; j < sz; j++) { - String hunkLine = hunkLines.get(j); - switch (hunkLine.charAt(0)) { + ByteBuffer hunkLine = hunkLines.get(j); + switch (hunkLine.array()[hunkLine.position()]) { case ' ': case '-': if (pos >= limit - || !newLines.get(pos).equals(hunkLine.substring(1))) { + || !newLines.get(pos).equals(slice(hunkLine, 1))) { return false; } pos++; @@ -721,13 +719,9 @@ private boolean canApplyAt(List hunkLines, List newLines, return true; } - private static boolean isChanged(List ol, List nl) { - if (ol.size() != nl.size()) - return true; - for (int i = 0; i < ol.size(); i++) - if (!ol.get(i).equals(nl.get(i))) - return true; - return false; + private ByteBuffer slice(ByteBuffer b, int off) { + int newOffset = b.position() + off; + return ByteBuffer.wrap(b.array(), newOffset, b.limit() - newOffset); } private boolean isNoNewlineAtEndOfFile(FileHeader fh) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java index 9f4b1fa49..d09da019d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2009, Google Inc. - * Copyright (C) 2008-2009, Johannes E. Schindelin and others + * Copyright (C) 2008-2021, Johannes E. Schindelin and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -16,6 +16,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import org.eclipse.jgit.errors.BinaryBlobException; import org.eclipse.jgit.errors.LargeObjectException; @@ -164,6 +165,27 @@ public String getString(int i) { return getString(i, i + 1, true); } + /** + * Get the raw text for a single line. + * + * @param i + * index of the line to extract. Note this is 0-based, so line + * number 1 is actually index 0. + * @return the text for the line, without a trailing LF, as a + * {@link ByteBuffer} that is backed by a slice of the + * {@link #getRawContent() raw content}, with the buffer's position + * on the start of the line and the limit at the end. + * @since 5.12 + */ + public ByteBuffer getRawString(int i) { + int s = getStart(i); + int e = getEnd(i); + if (e > 0 && content[e - 1] == '\n') { + e--; + } + return ByteBuffer.wrap(content, s, e - s); + } + /** * Get the text for a region of lines. * From 2a0295ccfd1cf8dcb1067575f38c86f430285cda Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Wed, 10 Mar 2021 18:04:25 +0100 Subject: [PATCH 7/8] ApplyCommand: handle completely empty context lines in text patches C git treats completely empty lines as empty context lines (which traditionally have a single blank). Apparently newer GNU diff may produce such lines; see [1]. ("Newer" meaning "since 2006"...) [1] https://github.com/git/git/commit/b507b465f7831 Change-Id: I80c1f030edb17a46289b1dabf11a2648d2660d38 Signed-off-by: Thomas Wolf --- .../org/eclipse/jgit/diff/emptyLine.patch | Bin 0 -> 134 bytes .../org/eclipse/jgit/diff/emptyLine_PostImage | Bin 0 -> 13 bytes .../org/eclipse/jgit/diff/emptyLine_PreImage | Bin 0 -> 13 bytes .../org/eclipse/jgit/api/ApplyCommandTest.java | 8 ++++++++ .../src/org/eclipse/jgit/api/ApplyCommand.java | 16 ++++++++++++++-- 5 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine_PreImage diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine.patch new file mode 100644 index 0000000000000000000000000000000000000000..18c80c4feb7b01d874e59d5fa7419798978da2df GIT binary patch literal 134 zcmY+5I}U>|3_y3E!ng6WS literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine_PostImage new file mode 100644 index 0000000000000000000000000000000000000000..45c2c9ba5bdb50a19c59911e40e6463d6e14dff2 GIT binary patch literal 13 UcmYex&*$PwN-W|^E6wEs032ci>Hq)$ literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/emptyLine_PreImage new file mode 100644 index 0000000000000000000000000000000000000000..1fd3fa23b0597045ca6507f6059fce4778e7f5e9 GIT binary patch literal 13 UcmYex&*$Pw%S`1;E6wEs032)s=>Px# literal 0 HcmV?d00001 diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java index b997ac009..807b96130 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ApplyCommandTest.java @@ -295,6 +295,14 @@ public void testEncodingChange() throws Exception { checkBinary("umlaut", true); } + @Test + public void testEmptyLine() throws Exception { + // C git accepts completely empty lines as empty context lines. + // According to comments in the C git sources (apply.c), newer GNU diff + // may produce such diffs. + checkBinary("emptyLine", true); + } + @Test public void testAddA1() throws Exception { ApplyResult result = init("A1", false, true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java index ec53412fc..f649c5fde 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java @@ -640,6 +640,11 @@ && canApplyAt(hunkLines, newLines, 0)) { int sz = hunkLines.size(); for (int j = 1; j < sz; j++) { ByteBuffer hunkLine = hunkLines.get(j); + if (!hunkLine.hasRemaining()) { + // Completely empty line; accept as empty context line + applyAt++; + continue; + } switch (hunkLine.array()[hunkLine.position()]) { case ' ': applyAt++; @@ -676,8 +681,7 @@ && canApplyAt(hunkLines, newLines, 0)) { // Must be the marker for the final newline break; } - out.write(line.array(), line.position(), - line.limit() - line.position()); + out.write(line.array(), line.position(), line.remaining()); if (l.hasNext()) { out.write('\n'); } @@ -703,6 +707,14 @@ private boolean canApplyAt(List hunkLines, int pos = line; for (int j = 1; j < sz; j++) { ByteBuffer hunkLine = hunkLines.get(j); + if (!hunkLine.hasRemaining()) { + // Empty line. Accept as empty context line. + if (pos >= limit || newLines.get(pos).hasRemaining()) { + return false; + } + pos++; + continue; + } switch (hunkLine.array()[hunkLine.position()]) { case ' ': case '-': From 1126f26d21b3aadf364af09c79d27d39de5e9bb9 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Wed, 10 Mar 2021 19:26:39 +0100 Subject: [PATCH 8/8] ApplyCommand: fix "no newline at end" detection Check the last line of the last hunk of a file, not the last line of the whole patch. Note that C git only checks that this line starts with "\ " and is at least 12 characters long because of possible different texts when non- English messages are used. Change-Id: I0db81699eb3e99ed7b536a3e2b8dc97df1f58a89 Signed-off-by: Thomas Wolf --- .../org/eclipse/jgit/diff/hello.patch | Bin 0 -> 269 bytes .../org/eclipse/jgit/diff/hello_PostImage | Bin 0 -> 3 bytes .../org/eclipse/jgit/diff/hello_PreImage | Bin 0 -> 5 bytes .../eclipse/jgit/api/ApplyCommandTest.java | 20 +++++++++++++++++- .../org/eclipse/jgit/api/ApplyCommand.java | 6 +++++- 5 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/hello.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/hello_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/hello_PreImage diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/hello.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/hello.patch new file mode 100644 index 0000000000000000000000000000000000000000..f015a38062dabcfff1cfc914bdc857ae2135a282 GIT binary patch literal 269 zcmaKmF$%*l5Cr>s#r8sqM2vAdy&x}0bFxl}K*){3@$U_m6XGV-EZpufF{cErpLqsf zUQ)`0&`7yPc_Z{`?8e?0%YlU%&f31~NNfFBkW%Wq@*dhjIe1Tce@GA9CsbKVt^%GR uHDXFgxd^GS%HKl#6