From efd1cc05af7e59a24763dfedb7fc44cda151be50 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Wed, 22 Jul 2020 00:51:24 +0200 Subject: [PATCH] Keep line endings for text files committed with CR/LF on text=auto Git never converts line endings if the version in the repository is a text file with CR/LF and text=auto. See [1]: "When the file has been committed with CRLF, no conversion is done." Because the sentence just before is about converting line endings on check-in, I had understood that in commit 60cf85a [2] to mean that no conversion on check-in was to be done. However, as bug 565048 and a code inspection of the C git code showed it really means no conversion is done on check-in *or check-out*. If the text attribute is not set but core.autocrlf = true, this is the same as text=auto eol=crlf. C git does not convert on check-out even on text=auto eol=lf if the index version is a text file with CR/LF. For check-in, one has to look at the intended target, which is done in WorkingTreeIterator since commit 60cf85a. For check-out, it can be done by looking at the source and can thus be done in the AutoLFOutputStream. Additionally, provide a constructor for AutoLFInputStream to do the same; for cases where the equivalent of a check-out is done via an input stream obtained from a blob. (EGit does that in its GitBlobStorage for the Eclipse compare framework; it's more efficient than using a TemporaryBuffer and DirCacheCheckout.getContent(), and it avoids the need for a temporary file.) Adapt existing tests, and add new checkout and merge tests to verify the resulting files have the correct line endings. EGit's GitBlobStorage will need to call the new version of EolStreamTypeUtil.wrapInputStream(). [1] https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-Settostringvalueauto [2] https://git.eclipse.org/r/c/jgit/jgit/+/127324 Bug: 565048 Change-Id: If1282ef43e2abd00263541bd10a01fe1f5c619fc Signed-off-by: Thomas Wolf --- .../jgit/api/EolStreamTypeUtilTest.java | 120 +++++++---------- .../jgit/lib/DirCacheCheckoutTest.java | 117 +++++++++++----- .../org/eclipse/jgit/merge/MergerTest.java | 44 +++++- .../jgit/util/io/AutoLFInputStreamTest.java | 43 ++++-- .../jgit/util/io/AutoLFOutputStreamTest.java | 84 ++++++++++++ .../jgit/util/io/AutoLFInputStream.java | 125 ++++++++++++++++-- .../jgit/util/io/AutoLFOutputStream.java | 63 +++------ .../jgit/util/io/EolStreamTypeUtil.java | 35 ++++- 8 files changed, 448 insertions(+), 183 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java index 1e3a39aad..2a553ce1a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java @@ -1,43 +1,11 @@ /* - * Copyright (C) 2015, Ivan Motsch + * Copyright (C) 2015, 2020 Ivan Motsch and others * - * This program and the accompanying materials are made available - * under the terms of the Eclipse Distribution License v1.0 which - * accompanies this distribution, is reproduced below, and is - * available at http://www.eclipse.org/org/documents/edl-v10.php + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the - * names of its contributors may be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.api; @@ -90,16 +58,20 @@ public void testCheckoutLF() throws Exception { testCheckout(TEXT_LF, AUTO_LF, "\r", "\r"); testCheckout(TEXT_LF, AUTO_LF, "\n", "\n"); - testCheckout(TEXT_LF, AUTO_LF, "\r\n", "\n"); + testCheckout(TEXT_LF, null, "\r\n", "\n"); + testCheckout(null, AUTO_LF, "\r\n", "\r\n"); testCheckout(TEXT_LF, AUTO_LF, "\n\r", "\n\r"); - testCheckout(TEXT_LF, AUTO_LF, "\n\r\n", "\n\n"); - testCheckout(TEXT_LF, AUTO_LF, "\r\n\r", "\n\r"); + testCheckout(TEXT_LF, null, "\n\r\n", "\n\n"); + testCheckout(null, AUTO_LF, "\n\r\n", "\n\r\n"); + testCheckout(TEXT_LF, null, "\r\n\r", "\n\r"); + testCheckout(null, AUTO_LF, "\r\n\r", "\r\n\r"); testCheckout(TEXT_LF, AUTO_LF, "a\nb\n", "a\nb\n"); testCheckout(TEXT_LF, AUTO_LF, "a\rb\r", "a\rb\r"); testCheckout(TEXT_LF, AUTO_LF, "a\n\rb\n\r", "a\n\rb\n\r"); - testCheckout(TEXT_LF, AUTO_LF, "a\r\nb\r\n", "a\nb\n"); + testCheckout(TEXT_LF, null, "a\r\nb\r\n", "a\nb\n"); + testCheckout(null, AUTO_LF, "a\r\nb\r\n", "a\r\nb\r\n"); } @Test @@ -153,45 +125,49 @@ private void testCheckout(EolStreamType streamTypeText, byte[] outputBytes = output.getBytes(UTF_8); byte[] expectedConversionBytes = expectedConversion.getBytes(UTF_8); - // test using output text and assuming it was declared TEXT - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeText)) { - out.write(outputBytes); + if (streamTypeText != null) { + // test using output text and assuming it was declared TEXT + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeText)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); } - assertArrayEquals(expectedConversionBytes, b.toByteArray()); - - // test using ouput text and assuming it was declared AUTO, using binary - // detection - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeWithBinaryCheck)) { - out.write(outputBytes); + if (streamTypeWithBinaryCheck != null) { + // test using output text and assuming it was declared AUTO, using + // binary detection + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeWithBinaryCheck)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); } - assertArrayEquals(expectedConversionBytes, b.toByteArray()); - // now pollute output text with some binary bytes outputBytes = extendWithBinaryData(outputBytes); expectedConversionBytes = extendWithBinaryData(expectedConversionBytes); - // again, test using output text and assuming it was declared TEXT - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeText)) { - out.write(outputBytes); + if (streamTypeText != null) { + // again, test using output text and assuming it was declared TEXT + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeText)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); } - assertArrayEquals(expectedConversionBytes, b.toByteArray()); - - // again, test using ouput text and assuming it was declared AUTO, using - // binary - // detection - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeWithBinaryCheck)) { - out.write(outputBytes); + if (streamTypeWithBinaryCheck != null) { + // again, test using output text and assuming it was declared AUTO, + // using binary detection + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeWithBinaryCheck)) { + out.write(outputBytes); + } + // expect no conversion + assertArrayEquals(outputBytes, b.toByteArray()); } - // expect no conversion - assertArrayEquals(outputBytes, b.toByteArray()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java index a272c8f2e..b943486b1 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java @@ -2,41 +2,13 @@ * Copyright (C) 2007, Dave Watson * Copyright (C) 2008-2011, Shawn O. Pearce * Copyright (C) 2008-2011, Robin Rosenberg - * Copyright (C) 2010-2011, Christian Halstrick - * and other copyright owners as documented in the project's IP log. + * Copyright (C) 2010, 2020 Christian Halstrick and others * * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v1.0 which accompanies this - * distribution, is reproduced below, and is available at - * http://www.eclipse.org/org/documents/edl-v10.php + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.lib; @@ -85,6 +57,7 @@ import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.StringUtils; import org.junit.Assume; import org.junit.Test; @@ -284,6 +257,86 @@ public void testInitialCheckout() throws Exception { } } + private void checkoutLineEndings(String inIndex, String expected, + String attributes) throws Exception { + try (Git git = new Git(db); + TestRepository db_t = new TestRepository<>(db)) { + BranchBuilder master = db_t.branch("master"); + master.commit().add("f", inIndex).message("m0").create(); + if (!StringUtils.isEmptyOrNull(attributes)) { + master.commit().add(".gitattributes", attributes) + .message("attributes").create(); + } + File f = new File(db.getWorkTree(), "f"); + assertFalse(f.exists()); + git.checkout().setName("master").call(); + assertTrue(f.exists()); + checkFile(f, expected); + } + } + + @Test + public void testCheckoutWithCRLF() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", null); + } + + @Test + public void testCheckoutWithCRLFAuto() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto"); + } + + @Test + public void testCheckoutWithCRLFAutoEolLf() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto eol=lf"); + } + + @Test + public void testCheckoutWithCRLFAutoEolNative() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto eol=native"); + } + + @Test + public void testCheckoutWithCRLFAutoEolCrLf() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto eol=crlf"); + } + + @Test + public void testCheckoutWithLF() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\nsecond line\n", null); + } + + @Test + public void testCheckoutWithLFAuto() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\nsecond line\n", "f text=auto"); + } + + @Test + public void testCheckoutWithLFAutoEolLf() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\nsecond line\n", "f text=auto eol=lf"); + } + + @Test + public void testCheckoutWithLFAutoEolNative() throws Exception { + checkoutLineEndings( + "first line\nsecond line\n", "first line\nsecond line\n" + .replaceAll("\n", System.lineSeparator()), + "f text=auto eol=native"); + } + + @Test + public void testCheckoutWithLFAutoEolCrLf() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\r\nsecond line\r\n", "f text=auto eol=crlf"); + } + private DirCacheCheckout resetHard(RevCommit commit) throws NoWorkTreeException, CorruptObjectException, IOException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java index 7a244e1d8..e2ac89be9 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, Robin Stocker and others + * Copyright (C) 2012, 2020 Robin Stocker and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -366,6 +366,48 @@ public void mergeWithCrlfInWT(MergeStrategy strategy) throws IOException, mergeResult.getMergeStatus()); } + @Theory + public void mergeConflictWithCrLfTextAuto(MergeStrategy strategy) + throws IOException, GitAPIException { + Git git = Git.wrap(db); + writeTrashFile("crlf.txt", "a crlf file\r\n"); + git.add().addFilepattern("crlf.txt").call(); + git.commit().setMessage("base").call(); + assertEquals("[crlf.txt, mode:100644, content:a crlf file\r\n]", + indexState(CONTENT)); + writeTrashFile(".gitattributes", "crlf.txt text=auto"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("attributes").call(); + + git.branchCreate().setName("brancha").call(); + + writeTrashFile("crlf.txt", "a crlf file\r\na second line\r\n"); + git.add().addFilepattern("crlf.txt").call(); + git.commit().setMessage("on master").call(); + assertEquals( + "[.gitattributes, mode:100644, content:crlf.txt text=auto]" + + "[crlf.txt, mode:100644, content:a crlf file\r\na second line\r\n]", + indexState(CONTENT)); + + git.checkout().setName("brancha").call(); + File testFile = writeTrashFile("crlf.txt", + "a crlf file\r\nanother line\r\n"); + git.add().addFilepattern("crlf.txt").call(); + git.commit().setMessage("on brancha").call(); + + MergeResult mergeResult = git.merge().setStrategy(strategy) + .include(db.resolve("master")).call(); + assertEquals(MergeResult.MergeStatus.CONFLICTING, + mergeResult.getMergeStatus()); + checkFile(testFile, + "a crlf file\r\n" // + + "<<<<<<< HEAD\n" // + + "another line\r\n" // + + "=======\n" // + + "a second line\r\n" // + + ">>>>>>> 8e9e704742f1bc8a41eac88aac4aeefd338b7384\n"); + } + @Theory public void mergeWithCrlfAutoCrlfTrue(MergeStrategy strategy) throws IOException, GitAPIException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java index fb90461e1..c391694fb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Marc Strapetz - * Copyright (C) 2015, Ivan Motsch and others + * Copyright (C) 2015, 2020 Ivan Motsch and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -17,7 +17,9 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.function.Function; +import org.eclipse.jgit.util.io.AutoLFInputStream.StreamFlag; import org.junit.Test; public class AutoLFInputStreamTest { @@ -25,47 +27,65 @@ public class AutoLFInputStreamTest { @Test public void testLF() throws IOException { final byte[] bytes = asBytes("1\n2\n3"); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testCR() throws IOException { final byte[] bytes = asBytes("1\r2\r3"); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testCRLF() throws IOException { - test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"), false); + test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3")); } @Test public void testLFCR() throws IOException { final byte[] bytes = asBytes("1\n\r2\n\r3"); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testEmpty() throws IOException { final byte[] bytes = asBytes(""); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testBinaryDetect() throws IOException { final byte[] bytes = asBytes("1\r\n2\r\n3\0"); - test(bytes, bytes, true); + test(bytes, bytes, StreamFlag.DETECT_BINARY); } @Test public void testBinaryDontDetect() throws IOException { - test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"), false); + test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0")); + } + + @Test + public void testCrLf() throws IOException { + byte[] bytes = asBytes("1\r\n2\n3\r\n\r"); + test(bytes, bytes, in -> AutoLFInputStream.create(in, + StreamFlag.DETECT_BINARY, StreamFlag.FOR_CHECKOUT)); + } + + @Test + public void testCrLfDontDetect() throws IOException { + test(asBytes("1\r\n2\r\n"), asBytes("1\n2\n"), + in -> AutoLFInputStream.create(in, StreamFlag.DETECT_BINARY)); + } + + private static void test(byte[] input, byte[] expected, StreamFlag... flags) + throws IOException { + test(input, expected, in -> AutoLFInputStream.create(in, flags)); } private static void test(byte[] input, byte[] expected, - boolean detectBinary) throws IOException { + Function factory) throws IOException { try (InputStream bis1 = new ByteArrayInputStream(input); - InputStream cis1 = new AutoLFInputStream(bis1, detectBinary)) { + InputStream cis1 = factory.apply(bis1)) { int index1 = 0; for (int b = cis1.read(); b != -1; b = cis1.read()) { assertEquals(expected[index1], (byte) b); @@ -77,8 +97,7 @@ private static void test(byte[] input, byte[] expected, for (int bufferSize = 1; bufferSize < 10; bufferSize++) { final byte[] buffer = new byte[bufferSize]; try (InputStream bis2 = new ByteArrayInputStream(input); - InputStream cis2 = new AutoLFInputStream(bis2, - detectBinary)) { + InputStream cis2 = factory.apply(bis2)) { int read = 0; for (int readNow = cis2.read(buffer, 0, diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java new file mode 100644 index 000000000..1b7e55e45 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2020, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.util.io; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.junit.Test; + +public class AutoLFOutputStreamTest { + + @Test + public void testLF() throws IOException { + final byte[] bytes = asBytes("1\n2\n3"); + test(bytes, bytes, false); + } + + @Test + public void testCR() throws IOException { + final byte[] bytes = asBytes("1\r2\r3"); + test(bytes, bytes, false); + } + + @Test + public void testCRLFNoDetect() throws IOException { + test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"), false); + } + + @Test + public void testLFCR() throws IOException { + final byte[] bytes = asBytes("1\n\r2\n\r3"); + test(bytes, bytes, false); + } + + @Test + public void testEmpty() throws IOException { + final byte[] bytes = asBytes(""); + test(bytes, bytes, false); + } + + @Test + public void testBinaryDetect() throws IOException { + final byte[] bytes = asBytes("1\r\n2\r\n3\0"); + test(bytes, bytes, true); + } + + @Test + public void testBinaryDontDetect() throws IOException { + test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"), false); + } + + @Test + public void testCrLfDetect() throws IOException { + byte[] bytes = asBytes("1\r\n2\n3\r\n\r"); + test(bytes, bytes, true); + } + + private static void test(byte[] input, byte[] expected, + boolean detectBinary) throws IOException { + try (ByteArrayOutputStream result = new ByteArrayOutputStream(); + OutputStream out = new AutoLFOutputStream(result, + detectBinary)) { + out.write(input); + out.close(); + assertArrayEquals(expected, result.toByteArray()); + } + } + + private static byte[] asBytes(String in) { + return in.getBytes(UTF_8); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java index 8c9b1bf5c..0e335a9dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, 2013 Marc Strapetz - * Copyright (C) 2015, Ivan Motsch and others + * Copyright (C) 2015, 2020 Ivan Motsch and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,26 +13,58 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Set; import org.eclipse.jgit.diff.RawText; /** * An InputStream that normalizes CRLF to LF. - * - * Existing single CR are not changed to LF, but retained as is. - * - * Optionally, a binary check on the first 8000 bytes is performed and in case - * of binary files, canonicalization is turned off (for the complete file). *

- * This is the former EolCanonicalizingInputStream with a new name in order to - * have same naming for all LF / CRLF streams + * Existing single CR are not changed to LF but are retained as is. + *

+ *

+ * Optionally, a binary check on the first 8kB is performed and in case of + * binary files, canonicalization is turned off (for the complete file). If + * binary checking determines that the input is CR/LF-delimited text and the + * stream has been created for checkout, canonicalization is also turned off. + *

* * @since 4.3 */ public class AutoLFInputStream extends InputStream { + + // This is the former EolCanonicalizingInputStream with a new name in order + // to have same naming for all LF / CRLF streams. + + /** + * Flags for controlling auto-detection of binary vs. text content (for + * text=auto). + * + * @since 5.9 + */ + public enum StreamFlag { + /** + * Check the first 8kB for binary content and switch off + * canonicalization off for the whole file if so. + */ + DETECT_BINARY, + /** + * If {@link #DETECT_BINARY} is set, throw an {@link IsBinaryException} + * if binary content is detected. + */ + ABORT_IF_BINARY, + /** + * If {@link #DETECT_BINARY} is set and content is found to be CR-LF + * delimited text, switch off canonicalization. + */ + FOR_CHECKOUT + } + private final byte[] single = new byte[1]; - private final byte[] buf = new byte[8096]; + private final byte[] buf = new byte[8 * 1024]; private final InputStream in; @@ -40,11 +72,23 @@ public class AutoLFInputStream extends InputStream { private int ptr; + /** + * Set to {@code true} if no CR/LF processing is to be done: if the input is + * binary data, or CR/LF-delimited text and {@link StreamFlag#FOR_CHECKOUT} + * was given. + */ + private boolean passAsIs; + + /** + * Set to {@code true} if the input was detected to be binary data. + */ private boolean isBinary; private boolean detectBinary; - private boolean abortIfBinary; + private final boolean abortIfBinary; + + private final boolean forCheckout; /** * A special exception thrown when {@link AutoLFInputStream} is told to @@ -62,20 +106,64 @@ public static class IsBinaryException extends IOException { } /** - * Creates a new InputStream, wrapping the specified stream + * Factory method for creating an {@link AutoLFInputStream} with the + * specified {@link StreamFlag flags}. + * + * @param in + * raw input stream + * @param flags + * {@link StreamFlag}s controlling the stream behavior + * @return a new {@link AutoLFInputStream} + * @since 5.9 + */ + public static AutoLFInputStream create(InputStream in, + StreamFlag... flags) { + if (flags == null) { + return new AutoLFInputStream(in, null); + } + EnumSet set = EnumSet.noneOf(StreamFlag.class); + set.addAll(Arrays.asList(flags)); + return new AutoLFInputStream(in, set); + } + + /** + * Creates a new InputStream, wrapping the specified stream. + * + * @param in + * raw input stream + * @param flags + * {@link StreamFlag}s controlling the stream behavior; + * {@code null} is treated as an empty set + * @since 5.9 + */ + public AutoLFInputStream(InputStream in, Set flags) { + this.in = in; + this.detectBinary = flags != null + && flags.contains(StreamFlag.DETECT_BINARY); + this.abortIfBinary = flags != null + && flags.contains(StreamFlag.ABORT_IF_BINARY); + this.forCheckout = flags != null + && flags.contains(StreamFlag.FOR_CHECKOUT); + } + + /** + * Creates a new InputStream, wrapping the specified stream. * * @param in * raw input stream * @param detectBinary * whether binaries should be detected * @since 2.0 + * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} + * instead */ + @Deprecated public AutoLFInputStream(InputStream in, boolean detectBinary) { this(in, detectBinary, false); } /** - * Creates a new InputStream, wrapping the specified stream + * Creates a new InputStream, wrapping the specified stream. * * @param in * raw input stream @@ -84,12 +172,16 @@ public AutoLFInputStream(InputStream in, boolean detectBinary) { * @param abortIfBinary * throw an IOException if the file is binary * @since 3.3 + * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} + * instead */ + @Deprecated public AutoLFInputStream(InputStream in, boolean detectBinary, boolean abortIfBinary) { this.in = in; this.detectBinary = detectBinary; this.abortIfBinary = abortIfBinary; + this.forCheckout = false; } /** {@inheritDoc} */ @@ -118,7 +210,7 @@ public int read(byte[] bs, int off, int len) } byte b = buf[ptr++]; - if (isBinary || b != '\r') { + if (passAsIs || b != '\r') { // Logic for binary files ends here bs[i++] = b; continue; @@ -170,9 +262,14 @@ private boolean fillBuffer() throws IOException { } if (detectBinary) { isBinary = RawText.isBinary(buf, cnt); + passAsIs = isBinary; detectBinary = false; - if (isBinary && abortIfBinary) + if (isBinary && abortIfBinary) { throw new IsBinaryException(); + } + if (!passAsIs && forCheckout) { + passAsIs = RawText.isCrLfText(buf, cnt); + } } ptr = 0; return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java index e235aa0ed..195fdb421 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java @@ -1,43 +1,12 @@ /* * Copyright (C) 2015, Ivan Motsch + * Copyright (C) 2020, Thomas Wolf and others * - * This program and the accompanying materials are made available - * under the terms of the Eclipse Distribution License v1.0 which - * accompanies this distribution, is reproduced below, and is - * available at http://www.eclipse.org/org/documents/edl-v10.php + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the - * names of its contributors may be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.util.io; @@ -49,11 +18,15 @@ /** * An OutputStream that reduces CRLF to LF. - * + *

* Existing single CR are not changed to LF, but retained as is. - * + *

+ *

* A binary check on the first 8000 bytes is performed and in case of binary - * files, canonicalization is turned off (for the complete file). + * files, canonicalization is turned off (for the complete file). If the binary + * check determines that the input is not binary but text with CR/LF, + * canonicalization is also turned off. + *

* * @since 4.3 */ @@ -76,9 +49,7 @@ public class AutoLFOutputStream extends OutputStream { private boolean isBinary; /** - *

* Constructor for AutoLFOutputStream. - *

* * @param out * an {@link java.io.OutputStream} object. @@ -88,9 +59,7 @@ public AutoLFOutputStream(OutputStream out) { } /** - *

* Constructor for AutoLFOutputStream. - *

* * @param out * an {@link java.io.OutputStream} object. @@ -123,14 +92,11 @@ public void write(byte[] b) throws IOException { public void write(byte[] b, int startOff, int startLen) throws IOException { final int overflow = buffer(b, startOff, startLen); - if (overflow < 0) { + if (overflow <= 0) { return; } final int off = startOff + startLen - overflow; final int len = overflow; - if (len == 0) { - return; - } int lastw = off; if (isBinary) { out.write(b, off, len); @@ -190,6 +156,9 @@ private int buffer(byte[] b, int off, int len) throws IOException { private void decideMode() throws IOException { if (detectBinary) { isBinary = RawText.isBinary(binbuf, binbufcnt); + if (!isBinary) { + isBinary = RawText.isCrLfText(binbuf, binbufcnt); + } detectBinary = false; } int cachedLen = binbufcnt; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java index c33c869b6..88ee2aee8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015, Ivan Motsch and others + * Copyright (C) 2015, 2020 Ivan Motsch and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,12 +12,14 @@ import java.io.InputStream; import java.io.OutputStream; +import java.util.EnumSet; import org.eclipse.jgit.attributes.Attributes; import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.treewalk.TreeWalk.OperationType; import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.util.SystemReader; +import org.eclipse.jgit.util.io.AutoLFInputStream.StreamFlag; /** * Utility used to create input and output stream wrappers for @@ -71,7 +73,7 @@ public static EolStreamType detectStreamType(OperationType op, /** * Wrap the input stream depending on - * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType} + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}. * * @param in * original stream @@ -82,15 +84,38 @@ public static EolStreamType detectStreamType(OperationType op, */ public static InputStream wrapInputStream(InputStream in, EolStreamType conversion) { + return wrapInputStream(in, conversion, false); + } + + /** + * Wrap the input stream depending on + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}. + * + * @param in + * original stream + * @param conversion + * to be performed + * @param forCheckout + * whether the stream is for checking out from the repository + * @return the converted stream depending on + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType} + * @since 5.9 + */ + public static InputStream wrapInputStream(InputStream in, + EolStreamType conversion, boolean forCheckout) { switch (conversion) { case TEXT_CRLF: return new AutoCRLFInputStream(in, false); case TEXT_LF: - return new AutoLFInputStream(in, false); + return AutoLFInputStream.create(in); case AUTO_CRLF: return new AutoCRLFInputStream(in, true); case AUTO_LF: - return new AutoLFInputStream(in, true); + EnumSet flags = forCheckout + ? EnumSet.of(StreamFlag.DETECT_BINARY, + StreamFlag.FOR_CHECKOUT) + : EnumSet.of(StreamFlag.DETECT_BINARY); + return new AutoLFInputStream(in, flags); default: return in; } @@ -98,7 +123,7 @@ public static InputStream wrapInputStream(InputStream in, /** * Wrap the output stream depending on - * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType} + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}. * * @param out * original stream