motiejus
/
jgit
1
Fork 0
Code Releases Activity

Keep line endings for text files committed with CR/LF on text=auto 

Git never converts line endings if the version in the repository is a
text file with CR/LF and text=auto. See [1]: "When the file has been
committed with CRLF, no conversion is done."

Because the sentence just before is about converting line endings on
check-in, I had understood that in commit 60cf85a [2] to mean that no
conversion on check-in was to be done. However, as bug 565048 and a
code inspection of the C git code showed it really means no conversion
is done on check-in *or check-out*.

If the text attribute is not set but core.autocrlf = true, this is
the same as text=auto eol=crlf. C git does not convert on check-out
even on text=auto eol=lf if the index version is a text file with
CR/LF.

For check-in, one has to look at the intended target, which is done
in WorkingTreeIterator since commit 60cf85a. For check-out, it can
be done by looking at the source and can thus be done in the
AutoLFOutputStream.

Additionally, provide a constructor for AutoLFInputStream to do
the same; for cases where the equivalent of a check-out is done via
an input stream obtained from a blob. (EGit does that in its
GitBlobStorage for the Eclipse compare framework; it's more efficient
than using a TemporaryBuffer and DirCacheCheckout.getContent(), and
it avoids the need for a temporary file.)

Adapt existing tests, and add new checkout and merge tests to verify
the resulting files have the correct line endings.

EGit's GitBlobStorage will need to call the new version of
EolStreamTypeUtil.wrapInputStream().

[1] https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-Settostringvalueauto
[2] https://git.eclipse.org/r/c/jgit/jgit/+/127324

Bug: 565048
Change-Id: If1282ef43e2abd00263541bd10a01fe1f5c619fc
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>
This commit is contained in:
Thomas Wolf 2020-07-22 00:51:24 +02:00
parent 71aeedb6ec
commit efd1cc05af
8 changed files with 448 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java
View File

@ -1,43 +1,11 @@
/*
* Copyright (C) 2015, Ivan Motsch <ivan.motsch@bsiag.com>
* Copyright (C) 2015, 2020 Ivan Motsch <ivan.motsch@bsiag.com> and others
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.api;
@ -90,16 +58,20 @@ public void testCheckoutLF() throws Exception {
testCheckout(TEXT_LF, AUTO_LF, "\r", "\r");
testCheckout(TEXT_LF, AUTO_LF, "\n", "\n");
testCheckout(TEXT_LF, AUTO_LF, "\r\n", "\n");
testCheckout(TEXT_LF, null, "\r\n", "\n");
testCheckout(null, AUTO_LF, "\r\n", "\r\n");
testCheckout(TEXT_LF, AUTO_LF, "\n\r", "\n\r");
testCheckout(TEXT_LF, AUTO_LF, "\n\r\n", "\n\n");
testCheckout(TEXT_LF, AUTO_LF, "\r\n\r", "\n\r");
testCheckout(TEXT_LF, null, "\n\r\n", "\n\n");
testCheckout(null, AUTO_LF, "\n\r\n", "\n\r\n");
testCheckout(TEXT_LF, null, "\r\n\r", "\n\r");
testCheckout(null, AUTO_LF, "\r\n\r", "\r\n\r");
testCheckout(TEXT_LF, AUTO_LF, "a\nb\n", "a\nb\n");
testCheckout(TEXT_LF, AUTO_LF, "a\rb\r", "a\rb\r");
testCheckout(TEXT_LF, AUTO_LF, "a\n\rb\n\r", "a\n\rb\n\r");
testCheckout(TEXT_LF, AUTO_LF, "a\r\nb\r\n", "a\nb\n");
testCheckout(TEXT_LF, null, "a\r\nb\r\n", "a\nb\n");
testCheckout(null, AUTO_LF, "a\r\nb\r\n", "a\r\nb\r\n");
}
@Test
@ -153,45 +125,49 @@ private void testCheckout(EolStreamType streamTypeText,
byte[] outputBytes = output.getBytes(UTF_8);
byte[] expectedConversionBytes = expectedConversion.getBytes(UTF_8);
// test using output text and assuming it was declared TEXT
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeText)) {
out.write(outputBytes);
if (streamTypeText != null) {
// test using output text and assuming it was declared TEXT
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeText)) {
out.write(outputBytes);
}
assertArrayEquals(expectedConversionBytes, b.toByteArray());
}
assertArrayEquals(expectedConversionBytes, b.toByteArray());
// test using ouput text and assuming it was declared AUTO, using binary
// detection
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeWithBinaryCheck)) {
out.write(outputBytes);
if (streamTypeWithBinaryCheck != null) {
// test using output text and assuming it was declared AUTO, using
// binary detection
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeWithBinaryCheck)) {
out.write(outputBytes);
}
assertArrayEquals(expectedConversionBytes, b.toByteArray());
}
assertArrayEquals(expectedConversionBytes, b.toByteArray());
// now pollute output text with some binary bytes
outputBytes = extendWithBinaryData(outputBytes);
expectedConversionBytes = extendWithBinaryData(expectedConversionBytes);
// again, test using output text and assuming it was declared TEXT
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeText)) {
out.write(outputBytes);
if (streamTypeText != null) {
// again, test using output text and assuming it was declared TEXT
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeText)) {
out.write(outputBytes);
}
assertArrayEquals(expectedConversionBytes, b.toByteArray());
}
assertArrayEquals(expectedConversionBytes, b.toByteArray());
// again, test using ouput text and assuming it was declared AUTO, using
// binary
// detection
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeWithBinaryCheck)) {
out.write(outputBytes);
if (streamTypeWithBinaryCheck != null) {
// again, test using output text and assuming it was declared AUTO,
// using binary detection
b = new ByteArrayOutputStream();
try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b,
streamTypeWithBinaryCheck)) {
out.write(outputBytes);
}
// expect no conversion
assertArrayEquals(outputBytes, b.toByteArray());
}
// expect no conversion
assertArrayEquals(outputBytes, b.toByteArray());
}
@Test

117
org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java
View File

@ -2,41 +2,13 @@
* Copyright (C) 2007, Dave Watson <dwatson@mimvista.com>
* Copyright (C) 2008-2011, Shawn O. Pearce <spearce@spearce.org>
* Copyright (C) 2008-2011, Robin Rosenberg <robin.rosenberg@dewire.com>
* Copyright (C) 2010-2011, Christian Halstrick <christian.halstrick@sap.com>
* and other copyright owners as documented in the project's IP log.
* Copyright (C) 2010, 2020 Christian Halstrick <christian.halstrick@sap.com> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v1.0 which accompanies this
* distribution, is reproduced below, and is available at
* http://www.eclipse.org/org/documents/edl-v10.php
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* - Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.lib;
@ -85,6 +57,7 @@
import org.eclipse.jgit.treewalk.WorkingTreeIterator;
import org.eclipse.jgit.util.FS;
import org.eclipse.jgit.util.FileUtils;
import org.eclipse.jgit.util.StringUtils;
import org.junit.Assume;
import org.junit.Test;
@ -284,6 +257,86 @@ public void testInitialCheckout() throws Exception {
}
}
private void checkoutLineEndings(String inIndex, String expected,
String attributes) throws Exception {
try (Git git = new Git(db);
TestRepository<Repository> db_t = new TestRepository<>(db)) {
BranchBuilder master = db_t.branch("master");
master.commit().add("f", inIndex).message("m0").create();
if (!StringUtils.isEmptyOrNull(attributes)) {
master.commit().add(".gitattributes", attributes)
.message("attributes").create();
}
File f = new File(db.getWorkTree(), "f");
assertFalse(f.exists());
git.checkout().setName("master").call();
assertTrue(f.exists());
checkFile(f, expected);
}
}
@Test
public void testCheckoutWithCRLF() throws Exception {
checkoutLineEndings("first line\r\nsecond line\r\n",
"first line\r\nsecond line\r\n", null);
}
@Test
public void testCheckoutWithCRLFAuto() throws Exception {
checkoutLineEndings("first line\r\nsecond line\r\n",
"first line\r\nsecond line\r\n", "f text=auto");
}
@Test
public void testCheckoutWithCRLFAutoEolLf() throws Exception {
checkoutLineEndings("first line\r\nsecond line\r\n",
"first line\r\nsecond line\r\n", "f text=auto eol=lf");
}
@Test
public void testCheckoutWithCRLFAutoEolNative() throws Exception {
checkoutLineEndings("first line\r\nsecond line\r\n",
"first line\r\nsecond line\r\n", "f text=auto eol=native");
}
@Test
public void testCheckoutWithCRLFAutoEolCrLf() throws Exception {
checkoutLineEndings("first line\r\nsecond line\r\n",
"first line\r\nsecond line\r\n", "f text=auto eol=crlf");
}
@Test
public void testCheckoutWithLF() throws Exception {
checkoutLineEndings("first line\nsecond line\n",
"first line\nsecond line\n", null);
}
@Test
public void testCheckoutWithLFAuto() throws Exception {
checkoutLineEndings("first line\nsecond line\n",
"first line\nsecond line\n", "f text=auto");
}
@Test
public void testCheckoutWithLFAutoEolLf() throws Exception {
checkoutLineEndings("first line\nsecond line\n",
"first line\nsecond line\n", "f text=auto eol=lf");
}
@Test
public void testCheckoutWithLFAutoEolNative() throws Exception {
checkoutLineEndings(
"first line\nsecond line\n", "first line\nsecond line\n"
.replaceAll("\n", System.lineSeparator()),
"f text=auto eol=native");
}
@Test
public void testCheckoutWithLFAutoEolCrLf() throws Exception {
checkoutLineEndings("first line\nsecond line\n",
"first line\r\nsecond line\r\n", "f text=auto eol=crlf");
}
private DirCacheCheckout resetHard(RevCommit commit)
throws NoWorkTreeException,
CorruptObjectException, IOException {

44
org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2012, Robin Stocker <robin@nibor.org> and others
* Copyright (C) 2012, 2020 Robin Stocker <robin@nibor.org> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@ -366,6 +366,48 @@ public void mergeWithCrlfInWT(MergeStrategy strategy) throws IOException,
mergeResult.getMergeStatus());
}
@Theory
public void mergeConflictWithCrLfTextAuto(MergeStrategy strategy)
throws IOException, GitAPIException {
Git git = Git.wrap(db);
writeTrashFile("crlf.txt", "a crlf file\r\n");
git.add().addFilepattern("crlf.txt").call();
git.commit().setMessage("base").call();
assertEquals("[crlf.txt, mode:100644, content:a crlf file\r\n]",
indexState(CONTENT));
writeTrashFile(".gitattributes", "crlf.txt text=auto");
git.add().addFilepattern(".gitattributes").call();
git.commit().setMessage("attributes").call();
git.branchCreate().setName("brancha").call();
writeTrashFile("crlf.txt", "a crlf file\r\na second line\r\n");
git.add().addFilepattern("crlf.txt").call();
git.commit().setMessage("on master").call();
assertEquals(
"[.gitattributes, mode:100644, content:crlf.txt text=auto]"
+ "[crlf.txt, mode:100644, content:a crlf file\r\na second line\r\n]",
indexState(CONTENT));
git.checkout().setName("brancha").call();
File testFile = writeTrashFile("crlf.txt",
"a crlf file\r\nanother line\r\n");
git.add().addFilepattern("crlf.txt").call();
git.commit().setMessage("on brancha").call();
MergeResult mergeResult = git.merge().setStrategy(strategy)
.include(db.resolve("master")).call();
assertEquals(MergeResult.MergeStatus.CONFLICTING,
mergeResult.getMergeStatus());
checkFile(testFile,
"a crlf file\r\n" //
+ "<<<<<<< HEAD\n" //
+ "another line\r\n" //
+ "=======\n" //
+ "a second line\r\n" //
+ ">>>>>>> 8e9e704742f1bc8a41eac88aac4aeefd338b7384\n");
}
@Theory
public void mergeWithCrlfAutoCrlfTrue(MergeStrategy strategy)
throws IOException, GitAPIException {

43
org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java
View File

@ -1,6 +1,6 @@
/*
* Copyright (C) 2010, Marc Strapetz <marc.strapetz@syntevo.com>
* Copyright (C) 2015, Ivan Motsch <ivan.motsch@bsiag.com> and others
* Copyright (C) 2015, 2020 Ivan Motsch <ivan.motsch@bsiag.com> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@ -17,7 +17,9 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.function.Function;
import org.eclipse.jgit.util.io.AutoLFInputStream.StreamFlag;
import org.junit.Test;
public class AutoLFInputStreamTest {
@ -25,47 +27,65 @@ public class AutoLFInputStreamTest {
@Test
public void testLF() throws IOException {
final byte[] bytes = asBytes("1\n2\n3");
test(bytes, bytes, false);
test(bytes, bytes);
}
@Test
public void testCR() throws IOException {
final byte[] bytes = asBytes("1\r2\r3");
test(bytes, bytes, false);
test(bytes, bytes);
}
@Test
public void testCRLF() throws IOException {
test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"), false);
test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"));
}
@Test
public void testLFCR() throws IOException {
final byte[] bytes = asBytes("1\n\r2\n\r3");
test(bytes, bytes, false);
test(bytes, bytes);
}
@Test
public void testEmpty() throws IOException {
final byte[] bytes = asBytes("");
test(bytes, bytes, false);
test(bytes, bytes);
}
@Test
public void testBinaryDetect() throws IOException {
final byte[] bytes = asBytes("1\r\n2\r\n3\0");
test(bytes, bytes, true);
test(bytes, bytes, StreamFlag.DETECT_BINARY);
}
@Test
public void testBinaryDontDetect() throws IOException {
test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"), false);
test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"));
}
@Test
public void testCrLf() throws IOException {
byte[] bytes = asBytes("1\r\n2\n3\r\n\r");
test(bytes, bytes, in -> AutoLFInputStream.create(in,
StreamFlag.DETECT_BINARY, StreamFlag.FOR_CHECKOUT));
}
@Test
public void testCrLfDontDetect() throws IOException {
test(asBytes("1\r\n2\r\n"), asBytes("1\n2\n"),
in -> AutoLFInputStream.create(in, StreamFlag.DETECT_BINARY));
}
private static void test(byte[] input, byte[] expected, StreamFlag... flags)
throws IOException {
test(input, expected, in -> AutoLFInputStream.create(in, flags));
}
private static void test(byte[] input, byte[] expected,
boolean detectBinary) throws IOException {
Function<InputStream, InputStream> factory) throws IOException {
try (InputStream bis1 = new ByteArrayInputStream(input);
InputStream cis1 = new AutoLFInputStream(bis1, detectBinary)) {
InputStream cis1 = factory.apply(bis1)) {
int index1 = 0;
for (int b = cis1.read(); b != -1; b = cis1.read()) {
assertEquals(expected[index1], (byte) b);
@ -77,8 +97,7 @@ private static void test(byte[] input, byte[] expected,
for (int bufferSize = 1; bufferSize < 10; bufferSize++) {
final byte[] buffer = new byte[bufferSize];
try (InputStream bis2 = new ByteArrayInputStream(input);
InputStream cis2 = new AutoLFInputStream(bis2,
detectBinary)) {
InputStream cis2 = factory.apply(bis2)) {
int read = 0;
for (int readNow = cis2.read(buffer, 0,

84
org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright (C) 2020, Thomas Wolf <thomas.wolf@paranor.ch> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.util.io;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertArrayEquals;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import org.junit.Test;
public class AutoLFOutputStreamTest {
@Test
public void testLF() throws IOException {
final byte[] bytes = asBytes("1\n2\n3");
test(bytes, bytes, false);
}
@Test
public void testCR() throws IOException {
final byte[] bytes = asBytes("1\r2\r3");
test(bytes, bytes, false);
}
@Test
public void testCRLFNoDetect() throws IOException {
test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"), false);
}
@Test
public void testLFCR() throws IOException {
final byte[] bytes = asBytes("1\n\r2\n\r3");
test(bytes, bytes, false);
}
@Test
public void testEmpty() throws IOException {
final byte[] bytes = asBytes("");
test(bytes, bytes, false);
}
@Test
public void testBinaryDetect() throws IOException {
final byte[] bytes = asBytes("1\r\n2\r\n3\0");
test(bytes, bytes, true);
}
@Test
public void testBinaryDontDetect() throws IOException {
test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"), false);
}
@Test
public void testCrLfDetect() throws IOException {
byte[] bytes = asBytes("1\r\n2\n3\r\n\r");
test(bytes, bytes, true);
}
private static void test(byte[] input, byte[] expected,
boolean detectBinary) throws IOException {
try (ByteArrayOutputStream result = new ByteArrayOutputStream();
OutputStream out = new AutoLFOutputStream(result,
detectBinary)) {
out.write(input);
out.close();
assertArrayEquals(expected, result.toByteArray());
}
}
private static byte[] asBytes(String in) {
return in.getBytes(UTF_8);
}
}

125
org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java
View File

@ -1,6 +1,6 @@
/*
* Copyright (C) 2010, 2013 Marc Strapetz <marc.strapetz@syntevo.com>
* Copyright (C) 2015, Ivan Motsch <ivan.motsch@bsiag.com> and others
* Copyright (C) 2015, 2020 Ivan Motsch <ivan.motsch@bsiag.com> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@ -13,26 +13,58 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Set;
import org.eclipse.jgit.diff.RawText;
/**
* An InputStream that normalizes CRLF to LF.
*
* Existing single CR are not changed to LF, but retained as is.
*
* Optionally, a binary check on the first 8000 bytes is performed and in case
* of binary files, canonicalization is turned off (for the complete file).
* <p>
* This is the former EolCanonicalizingInputStream with a new name in order to
* have same naming for all LF / CRLF streams
* Existing single CR are not changed to LF but are retained as is.
* </p>
* <p>
* Optionally, a binary check on the first 8kB is performed and in case of
* binary files, canonicalization is turned off (for the complete file). If
* binary checking determines that the input is CR/LF-delimited text and the
* stream has been created for checkout, canonicalization is also turned off.
* </p>
*
* @since 4.3
*/
public class AutoLFInputStream extends InputStream {
// This is the former EolCanonicalizingInputStream with a new name in order
// to have same naming for all LF / CRLF streams.
/**
* Flags for controlling auto-detection of binary vs. text content (for
* text=auto).
*
* @since 5.9
*/
public enum StreamFlag {
/**
* Check the first 8kB for binary content and switch off
* canonicalization off for the whole file if so.
*/
DETECT_BINARY,
/**
* If {@link #DETECT_BINARY} is set, throw an {@link IsBinaryException}
* if binary content is detected.
*/
ABORT_IF_BINARY,
/**
* If {@link #DETECT_BINARY} is set and content is found to be CR-LF
* delimited text, switch off canonicalization.
*/
FOR_CHECKOUT
}
private final byte[] single = new byte[1];
private final byte[] buf = new byte[8096];
private final byte[] buf = new byte[8 * 1024];
private final InputStream in;
@ -40,11 +72,23 @@ public class AutoLFInputStream extends InputStream {
private int ptr;
/**
* Set to {@code true} if no CR/LF processing is to be done: if the input is
* binary data, or CR/LF-delimited text and {@link StreamFlag#FOR_CHECKOUT}
* was given.
*/
private boolean passAsIs;
/**
* Set to {@code true} if the input was detected to be binary data.
*/
private boolean isBinary;
private boolean detectBinary;
private boolean abortIfBinary;
private final boolean abortIfBinary;
private final boolean forCheckout;
/**
* A special exception thrown when {@link AutoLFInputStream} is told to
@ -62,20 +106,64 @@ public static class IsBinaryException extends IOException {
}
/**
* Creates a new InputStream, wrapping the specified stream
* Factory method for creating an {@link AutoLFInputStream} with the
* specified {@link StreamFlag flags}.
*
* @param in
* raw input stream
* @param flags
* {@link StreamFlag}s controlling the stream behavior
* @return a new {@link AutoLFInputStream}
* @since 5.9
*/
public static AutoLFInputStream create(InputStream in,
StreamFlag... flags) {
if (flags == null) {
return new AutoLFInputStream(in, null);
}
EnumSet<StreamFlag> set = EnumSet.noneOf(StreamFlag.class);
set.addAll(Arrays.asList(flags));
return new AutoLFInputStream(in, set);
}
/**
* Creates a new InputStream, wrapping the specified stream.
*
* @param in
* raw input stream
* @param flags
* {@link StreamFlag}s controlling the stream behavior;
* {@code null} is treated as an empty set
* @since 5.9
*/
public AutoLFInputStream(InputStream in, Set<StreamFlag> flags) {
this.in = in;
this.detectBinary = flags != null
&& flags.contains(StreamFlag.DETECT_BINARY);
this.abortIfBinary = flags != null
&& flags.contains(StreamFlag.ABORT_IF_BINARY);
this.forCheckout = flags != null
&& flags.contains(StreamFlag.FOR_CHECKOUT);
}
/**
* Creates a new InputStream, wrapping the specified stream.
*
* @param in
* raw input stream
* @param detectBinary
* whether binaries should be detected
* @since 2.0
* @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)}
* instead
*/
@Deprecated
public AutoLFInputStream(InputStream in, boolean detectBinary) {
this(in, detectBinary, false);
}
/**
* Creates a new InputStream, wrapping the specified stream
* Creates a new InputStream, wrapping the specified stream.
*
* @param in
* raw input stream
@ -84,12 +172,16 @@ public AutoLFInputStream(InputStream in, boolean detectBinary) {
* @param abortIfBinary
* throw an IOException if the file is binary
* @since 3.3
* @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)}
* instead
*/
@Deprecated
public AutoLFInputStream(InputStream in, boolean detectBinary,
boolean abortIfBinary) {
this.in = in;
this.detectBinary = detectBinary;
this.abortIfBinary = abortIfBinary;
this.forCheckout = false;
}
/** {@inheritDoc} */
@ -118,7 +210,7 @@ public int read(byte[] bs, int off, int len)
}
byte b = buf[ptr++];
if (isBinary || b != '\r') {
if (passAsIs || b != '\r') {
// Logic for binary files ends here
bs[i++] = b;
continue;
@ -170,9 +262,14 @@ private boolean fillBuffer() throws IOException {
}
if (detectBinary) {
isBinary = RawText.isBinary(buf, cnt);
passAsIs = isBinary;
detectBinary = false;
if (isBinary && abortIfBinary)
if (isBinary && abortIfBinary) {
throw new IsBinaryException();
}
if (!passAsIs && forCheckout) {
passAsIs = RawText.isCrLfText(buf, cnt);
}
}
ptr = 0;
return true;

63
org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java
View File

@ -1,43 +1,12 @@
/*
* Copyright (C) 2015, Ivan Motsch <ivan.motsch@bsiag.com>
* Copyright (C) 2020, Thomas Wolf <thomas.wolf@paranor.ch> and others
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.util.io;
@ -49,11 +18,15 @@
/**
* An OutputStream that reduces CRLF to LF.
*
* <p>
* Existing single CR are not changed to LF, but retained as is.
*
* </p>
* <p>
* A binary check on the first 8000 bytes is performed and in case of binary
* files, canonicalization is turned off (for the complete file).
* files, canonicalization is turned off (for the complete file). If the binary
* check determines that the input is not binary but text with CR/LF,
* canonicalization is also turned off.
* </p>
*
* @since 4.3
*/
@ -76,9 +49,7 @@ public class AutoLFOutputStream extends OutputStream {
private boolean isBinary;
/**
* <p>
* Constructor for AutoLFOutputStream.
* </p>
*
* @param out
* an {@link java.io.OutputStream} object.
@ -88,9 +59,7 @@ public AutoLFOutputStream(OutputStream out) {
}
/**
* <p>
* Constructor for AutoLFOutputStream.
* </p>
*
* @param out
* an {@link java.io.OutputStream} object.
@ -123,14 +92,11 @@ public void write(byte[] b) throws IOException {
public void write(byte[] b, int startOff, int startLen)
throws IOException {
final int overflow = buffer(b, startOff, startLen);
if (overflow < 0) {
if (overflow <= 0) {
return;
}
final int off = startOff + startLen - overflow;
final int len = overflow;
if (len == 0) {
return;
}
int lastw = off;
if (isBinary) {
out.write(b, off, len);
@ -190,6 +156,9 @@ private int buffer(byte[] b, int off, int len) throws IOException {
private void decideMode() throws IOException {
if (detectBinary) {
isBinary = RawText.isBinary(binbuf, binbufcnt);
if (!isBinary) {
isBinary = RawText.isCrLfText(binbuf, binbufcnt);
}
detectBinary = false;
}
int cachedLen = binbufcnt;

35
org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2015, Ivan Motsch <ivan.motsch@bsiag.com> and others
* Copyright (C) 2015, 2020 Ivan Motsch <ivan.motsch@bsiag.com> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
@ -12,12 +12,14 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.EnumSet;
import org.eclipse.jgit.attributes.Attributes;
import org.eclipse.jgit.lib.CoreConfig.EolStreamType;
import org.eclipse.jgit.treewalk.TreeWalk.OperationType;
import org.eclipse.jgit.treewalk.WorkingTreeOptions;
import org.eclipse.jgit.util.SystemReader;
import org.eclipse.jgit.util.io.AutoLFInputStream.StreamFlag;
/**
* Utility used to create input and output stream wrappers for
@ -71,7 +73,7 @@ public static EolStreamType detectStreamType(OperationType op,
/**
* Wrap the input stream depending on
* {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}
* {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}.
*
* @param in
* original stream
@ -82,15 +84,38 @@ public static EolStreamType detectStreamType(OperationType op,
*/
public static InputStream wrapInputStream(InputStream in,
EolStreamType conversion) {
return wrapInputStream(in, conversion, false);
}
/**
* Wrap the input stream depending on
* {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}.
*
* @param in
* original stream
* @param conversion
* to be performed
* @param forCheckout
* whether the stream is for checking out from the repository
* @return the converted stream depending on
* {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}
* @since 5.9
*/
public static InputStream wrapInputStream(InputStream in,
EolStreamType conversion, boolean forCheckout) {
switch (conversion) {
case TEXT_CRLF:
return new AutoCRLFInputStream(in, false);
case TEXT_LF:
return new AutoLFInputStream(in, false);
return AutoLFInputStream.create(in);
case AUTO_CRLF:
return new AutoCRLFInputStream(in, true);
case AUTO_LF:
return new AutoLFInputStream(in, true);
EnumSet<StreamFlag> flags = forCheckout
? EnumSet.of(StreamFlag.DETECT_BINARY,
StreamFlag.FOR_CHECKOUT)
: EnumSet.of(StreamFlag.DETECT_BINARY);
return new AutoLFInputStream(in, flags);
default:
return in;
}
@ -98,7 +123,7 @@ public static InputStream wrapInputStream(InputStream in,
/**
* Wrap the output stream depending on
* {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}
* {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}.
*
* @param out
* original stream