From f6c989665ed768022c4058c227c54685d268237d Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 17 Jun 2019 00:34:49 +0200 Subject: [PATCH 01/47] Fix API problem filters Change-Id: I0f7a72934022346c9903d7dc5ec11e959cc8dac3 Signed-off-by: Matthias Sohn --- org.eclipse.jgit.lfs.server/.settings/.api_filters | 2 +- org.eclipse.jgit/.settings/.api_filters | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.eclipse.jgit.lfs.server/.settings/.api_filters b/org.eclipse.jgit.lfs.server/.settings/.api_filters index 84e6b19b6..727953ea6 100644 --- a/org.eclipse.jgit.lfs.server/.settings/.api_filters +++ b/org.eclipse.jgit.lfs.server/.settings/.api_filters @@ -3,7 +3,7 @@ - + diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 524c59191..a8404dd59 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -3,7 +3,7 @@ - + From 6d0168a4147226453cb1add5c278018d4603fcc4 Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Tue, 25 Sep 2018 14:11:23 +0900 Subject: [PATCH 02/47] Make inner classes static where possible As reported by Error Prone: An inner class should be static unless it references members of its enclosing class. An inner class that is made non-static unnecessarily uses more memory and does not make the intent of the class clear. See https://errorprone.info/bugpattern/ClassCanBeStatic Change-Id: Ib99d120532630dba63cf400cc1c61c318286fc41 Signed-off-by: David Pursehouse (cherry picked from commit ee40efcea44bc0c9a28afe29a80c87636947484e) --- .../jgit/transport/http/apache/HttpClientConnectionTest.java | 2 +- .../src/org/eclipse/jgit/junit/MockSystemReader.java | 2 +- .../src/org/eclipse/jgit/pgm/debug/TextHashFunctions.java | 2 +- .../tst/org/eclipse/jgit/api/ArchiveCommandTest.java | 5 +++-- .../tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java | 2 +- .../jgit/internal/storage/file/WindowCacheGetTest.java | 2 +- .../tst/org/eclipse/jgit/merge/MergerTest.java | 4 ++-- .../tst/org/eclipse/jgit/revplot/PlotCommitListTest.java | 2 +- .../org/eclipse/jgit/treewalk/AbstractTreeIteratorTest.java | 2 +- .../org/eclipse/jgit/util/io/TimeoutOutputStreamTest.java | 2 +- .../jgit/internal/storage/file/PackBitmapIndexRemapper.java | 2 +- org.eclipse.jgit/src/org/eclipse/jgit/lib/RefUpdate.java | 2 +- 12 files changed, 15 insertions(+), 14 deletions(-) diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/transport/http/apache/HttpClientConnectionTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/transport/http/apache/HttpClientConnectionTest.java index 72c4921de..8a0d59c74 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/transport/http/apache/HttpClientConnectionTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/transport/http/apache/HttpClientConnectionTest.java @@ -76,7 +76,7 @@ public void testGetHeaderFieldsAllowMultipleValues() assertTrue(headerValues.contains("NTLM")); } - private class HttpResponseMock extends AbstractHttpMessage + private static class HttpResponseMock extends AbstractHttpMessage implements HttpResponse { @Override public StatusLine getStatusLine() { diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java index d3d7d682e..8de386e34 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java @@ -70,7 +70,7 @@ * Mock {@link org.eclipse.jgit.util.SystemReader} for tests. */ public class MockSystemReader extends SystemReader { - private final class MockConfig extends FileBasedConfig { + private static final class MockConfig extends FileBasedConfig { private MockConfig(File cfgLocation, FS fs) { super(cfgLocation, fs); } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/TextHashFunctions.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/TextHashFunctions.java index bb51b5048..300a01d24 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/TextHashFunctions.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/TextHashFunctions.java @@ -474,7 +474,7 @@ private static abstract class Fold { } /** Utility to help us identify unique lines in a file. */ - private class Line { + private static class Line { private final RawText txt; private final int pos; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java index 1300f98d8..4883bcacc 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java @@ -190,7 +190,8 @@ public void archiveByDirectoryPath() throws GitAPIException, IOException { } } - private class MockFormat implements ArchiveCommand.Format { + private static class MockFormat + implements ArchiveCommand.Format { private Map entries = new HashMap<>(); @@ -240,7 +241,7 @@ public Iterable suffixes() { } } - public class MockOutputStream extends OutputStream { + public static class MockOutputStream extends OutputStream { private int foo; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java index 92f0cf619..f23e4be0a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java @@ -139,7 +139,7 @@ public void setUp() throws Exception { resolveRelativeUris(); } - class IndexedRepos implements RepoCommand.RemoteReader { + static class IndexedRepos implements RepoCommand.RemoteReader { Map uriRepoMap; IndexedRepos() { uriRepoMap = new HashMap<>(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java index 01426eeb8..9063b6518 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/WindowCacheGetTest.java @@ -141,7 +141,7 @@ private void doCacheTests() throws IOException { } } - private class TestObject { + private static class TestObject { ObjectId id; int type; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java index 8f12dd7b2..7f5dba697 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java @@ -839,7 +839,7 @@ public ObjectReader newReader() { /** * Throws an exception if reading beyond limit. */ - class BigReadForbiddenStream extends ObjectStream.Filter { + static class BigReadForbiddenStream extends ObjectStream.Filter { int limit; BigReadForbiddenStream(ObjectStream orig, int limit) { @@ -878,7 +878,7 @@ public int read(byte[] b, int off, int len) throws IOException { } } - class BigReadForbiddenReader extends ObjectReader.Filter { + static class BigReadForbiddenReader extends ObjectReader.Filter { ObjectReader delegate; int limit; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revplot/PlotCommitListTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revplot/PlotCommitListTest.java index 9a6043f32..7297de364 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revplot/PlotCommitListTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revplot/PlotCommitListTest.java @@ -56,7 +56,7 @@ public class PlotCommitListTest extends RevWalkTestCase { - class CommitListAssert { + static class CommitListAssert { private PlotCommitList pcl; private PlotCommit current; private int nextIndex = 0; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/AbstractTreeIteratorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/AbstractTreeIteratorTest.java index 934984f7e..8afd49ab4 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/AbstractTreeIteratorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/AbstractTreeIteratorTest.java @@ -66,7 +66,7 @@ private static String prefix(String path) { return s > 0 ? path.substring(0, s) : ""; } - public class FakeTreeIterator extends WorkingTreeIterator { + public static class FakeTreeIterator extends WorkingTreeIterator { public FakeTreeIterator(String pathName, FileMode fileMode) { super(prefix(pathName), new Config().get(WorkingTreeOptions.KEY)); mode = fileMode.getBits(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/TimeoutOutputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/TimeoutOutputStreamTest.java index a63b1cb02..c35f90ca7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/TimeoutOutputStreamTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/TimeoutOutputStreamTest.java @@ -282,7 +282,7 @@ private static long now() { return System.currentTimeMillis(); } - private final class FullPipeInputStream extends PipedInputStream { + private static final class FullPipeInputStream extends PipedInputStream { FullPipeInputStream(PipedOutputStream src) throws IOException { super(src); src.write(new byte[PIPE_SIZE]); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java index 70695880d..964cc4d12 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java @@ -207,7 +207,7 @@ public EWAHCompressedBitmap getBitmap(AnyObjectId objectId) { } /** An entry in the old PackBitmapIndex. */ - public final class Entry extends ObjectId { + public static final class Entry extends ObjectId { private final int flags; Entry(AnyObjectId src, int flags) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefUpdate.java index 5cd593ed5..fc3ea8467 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefUpdate.java @@ -823,7 +823,7 @@ private static RevObject safeParseOld(RevWalk rw, AnyObjectId oldId) * Handle the abstraction of storing a ref update. This is because both * updating and deleting of a ref have merge testing in common. */ - private abstract class Store { + private static abstract class Store { abstract Result execute(Result status) throws IOException; } } From 2d0a1adf0529d5d23c7752437d664784a2e4a578 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 18 Jun 2019 16:55:02 +0200 Subject: [PATCH 03/47] Update Maven plugins and cleanup Maven warnings update Maven plugins - jacoco-maven-plugin to 0.8.4 - japicmp-maven-plugin to 0.14.1 - maven-compiler-plugin to 3.8.1 - maven-deploy-plugin to 3.0.0-M1 - maven-enforcer-plugin to 3.0.0-M2 - maven-install-plugin to 3.0.0-M1 - maven-jar-plugin to 3.1.2 - maven-javadoc-plugin to 3.1.0 - maven-jxr-plugin to 3.0.0 - maven-pmd-plugin to 3.12.0 - maven-resources-plugin to 3.1.0 - maven-shade-plugin to 3.2.1 - maven-source-plugin to 3.1.0 - maven-surefire-plugin to 3.0.0-M3 - spotbugs-maven-plugin to 3.1.12 - tycho to 1.3.0 - tycho-pack200a-plugin to 1.3.0 - tycho-pack200b-plugin to 1.3.0 Cleanup Maven warnings - pin version of all used Maven plugins - remove deprecated way to declare minimum Maven version Change-Id: If23e2e2bb03e5e1e7b1eb9d4924a8faa0aa3704e Signed-off-by: Matthias Sohn --- org.eclipse.jgit.packaging/pom.xml | 40 ++++++++++++++++++++--- org.eclipse.jgit.test/pom.xml | 1 + org.eclipse.jgit/pom.xml | 1 + pom.xml | 51 +++++++++++++++++++++--------- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/org.eclipse.jgit.packaging/pom.xml b/org.eclipse.jgit.packaging/pom.xml index d763cb03b..879755e15 100644 --- a/org.eclipse.jgit.packaging/pom.xml +++ b/org.eclipse.jgit.packaging/pom.xml @@ -47,10 +47,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - - 3.0 - - org.eclipse.jgit jgit.tycho.parent 5.1.9-SNAPSHOT @@ -59,7 +55,7 @@ JGit Tycho Parent - 1.2.0 + 1.3.0 ${tycho-version} jgit-4.6 @@ -223,6 +219,21 @@ tycho-p2-plugin ${tycho-version} + + org.eclipse.tycho + tycho-p2-publisher-plugin + ${tycho-version} + + + org.eclipse.tycho + tycho-p2-repository-plugin + ${tycho-version} + + + org.eclipse.tycho + tycho-packaging-plugin + ${tycho-version} + org.eclipse.tycho.extras tycho-pack200a-plugin @@ -243,6 +254,25 @@ build-helper-maven-plugin 3.0.0 + + maven-clean-plugin + 3.1.0 + + + org.apache.maven.plugins + maven-deploy-plugin + 3.0.0-M1 + + + org.apache.maven.plugins + maven-install-plugin + 3.0.0-M1 + + + org.apache.maven.plugins + maven-site-plugin + 3.7.1 + diff --git a/org.eclipse.jgit.test/pom.xml b/org.eclipse.jgit.test/pom.xml index 7a4d58017..f42eb54ed 100644 --- a/org.eclipse.jgit.test/pom.xml +++ b/org.eclipse.jgit.test/pom.xml @@ -123,6 +123,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven-surefire-plugin-version} -Djgit.test.long=true diff --git a/org.eclipse.jgit/pom.xml b/org.eclipse.jgit/pom.xml index eedb5e80d..cb35c2fb6 100644 --- a/org.eclipse.jgit/pom.xml +++ b/org.eclipse.jgit/pom.xml @@ -209,6 +209,7 @@ com.github.spotbugs spotbugs-maven-plugin + ${spotbugs-maven-plugin-version} findBugs/FindBugsExcludeFilter.xml diff --git a/pom.xml b/pom.xml index 934f1b895..a1e1bae91 100644 --- a/pom.xml +++ b/pom.xml @@ -208,17 +208,18 @@ 4.3.1 3.1.0 9.4.11.v20180605 - 0.13.0 + 0.14.1 4.5.5 4.4.9 1.7.2 1.2.15 - 3.0.1 - 1.2.0 + 3.1.0 + 1.3.0 2.8.2 - 3.1.6 - 2.21.0 - 3.8.0 + 3.1.12 + 3.0.0-M3 + ${maven-surefire-plugin-version} + 3.8.1 jacoco @@ -250,7 +251,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.1.0 + 3.1.2 @@ -275,7 +276,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.1.1 + 3.2.1 @@ -293,7 +294,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.1.0 @@ -305,7 +306,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.21.0 + ${maven-surefire-plugin-version} ${test-fork-count} true @@ -338,7 +339,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.10.0 + 3.12.0 utf-8 100 @@ -376,7 +377,7 @@ org.jacoco jacoco-maven-plugin - 0.8.1 + 0.8.4 org.apache.maven.plugins @@ -398,12 +399,32 @@ org.apache.maven.plugins maven-jxr-plugin - 2.5 + 3.0.0 org.apache.maven.plugins maven-project-info-reports-plugin - 2.9 + 3.0.0 + + + org.apache.maven.plugins + maven-deploy-plugin + 3.0.0-M1 + + + org.apache.maven.plugins + maven-install-plugin + 3.0.0-M1 + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin-version} + + + org.apache.maven.plugins + maven-resources-plugin + 3.1.0 @@ -412,7 +433,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M1 + 3.0.0-M2 enforce-maven From c366f6483925f19670dae72c6b775911772f1ae6 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 18 Jun 2019 16:55:02 +0200 Subject: [PATCH 04/47] Update Maven plugins ecj, plexus, error-prone update Maven plugins - ecj to 3.17.0 - error_prone_core to 2.3.3 - plexus-compiler-eclipse to 2.8.5 - plexus-compiler-javac to 2.8.5 - plexus-compiler-javac-errorprone to 2.8.5 Change-Id: I51ecb44538915ed84db041510562394bce977a3e Signed-off-by: Matthias Sohn --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index a1e1bae91..b7e33305c 100644 --- a/pom.xml +++ b/pom.xml @@ -807,19 +807,19 @@ org.codehaus.plexus plexus-compiler-javac - 2.8.4 + 2.8.5 org.codehaus.plexus plexus-compiler-javac-errorprone - 2.8.4 + 2.8.5 com.google.errorprone error_prone_core - 2.3.1 + 2.3.3 @@ -852,12 +852,12 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.8.4 + 2.8.5 org.eclipse.jdt ecj - 3.13.102 + 3.17.0 From 9387288a8612eb9e9478c5ecfe4b1351a0ed4717 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 16 Jun 2019 23:58:06 +0200 Subject: [PATCH 05/47] Fix non-deterministic hash of archives created by ArchiveCommand Archives created by the ArchiveCommand didn't produce deterministic archive hashes. For RevCommits RevWalk.parseTree returns the root tree instead of the RevCommit hence retrieving the commit's timestamp didn't work. Instead use RevWalk.parseAny and extract the tree manually. Archive entries store timestamps with 1 second resolution hence we need to wait longer when creating the same archive twice and compare archive hashes. Otherwise hash comparison in tests wouldn't fail without this patch. Bug: 548312 Change-Id: I437d515de51cf68265584d28a8446cebe6341b79 Signed-off-by: Matthias Sohn --- org.eclipse.jgit.test/META-INF/MANIFEST.MF | 8 + org.eclipse.jgit.test/pom.xml | 6 + .../eclipse/jgit/api/ArchiveCommandTest.java | 191 +++++++++++++++++- .../org/eclipse/jgit/api/ArchiveCommand.java | 35 +++- 4 files changed, 226 insertions(+), 14 deletions(-) diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index 69ea99b4f..f469173aa 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -10,8 +10,15 @@ Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", com.jcraft.jsch;version="[0.1.54,0.2.0)", + org.apache.commons.compress.archivers;version="[1.15.0,2.0)", + org.apache.commons.compress.archivers.tar;version="[1.15.0,2.0)", + org.apache.commons.compress.archivers.zip;version="[1.15.0,2.0)", + org.apache.commons.compress.compressors.bzip2;version="[1.15.0,2.0)", + org.apache.commons.compress.compressors.gzip;version="[1.15.0,2.0)", + org.apache.commons.compress.compressors.xz;version="[1.15.0,2.0)", org.eclipse.jgit.api;version="[5.1.9,5.2.0)", org.eclipse.jgit.api.errors;version="[5.1.9,5.2.0)", + org.eclipse.jgit.archive;version="[5.1.9,5.2.0)", org.eclipse.jgit.attributes;version="[5.1.9,5.2.0)", org.eclipse.jgit.awtui;version="[5.1.9,5.2.0)", org.eclipse.jgit.blame;version="[5.1.9,5.2.0)", @@ -55,6 +62,7 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", org.eclipse.jgit.util;version="[5.1.9,5.2.0)", org.eclipse.jgit.util.io;version="[5.1.9,5.2.0)", org.eclipse.jgit.util.sha1;version="[5.1.9,5.2.0)", + org.tukaani.xz;version="[1.6.0,2.0)", org.junit;version="[4.12,5.0.0)", org.junit.experimental.theories;version="[4.12,5.0.0)", org.junit.rules;version="[4.12,5.0.0)", diff --git a/org.eclipse.jgit.test/pom.xml b/org.eclipse.jgit.test/pom.xml index f42eb54ed..3a887efb8 100644 --- a/org.eclipse.jgit.test/pom.xml +++ b/org.eclipse.jgit.test/pom.xml @@ -112,6 +112,12 @@ org.eclipse.jgit.pgm ${project.version} + + + org.tukaani + xz + true + diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java index 4883bcacc..fbec024a8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java @@ -46,20 +46,44 @@ import static org.junit.Assert.assertNull; import java.beans.Statement; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; +import org.eclipse.jgit.api.errors.AbortedByHookException; +import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.NoFilepatternException; +import org.eclipse.jgit.api.errors.NoHeadException; +import org.eclipse.jgit.api.errors.NoMessageException; +import org.eclipse.jgit.api.errors.UnmergedPathsException; +import org.eclipse.jgit.api.errors.WrongRepositoryStateException; +import org.eclipse.jgit.archive.ArchiveFormats; +import org.eclipse.jgit.errors.AmbiguousObjectException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.StringUtils; import org.junit.After; import org.junit.Before; @@ -67,9 +91,14 @@ public class ArchiveCommandTest extends RepositoryTestCase { + // archives store timestamp with 1 second resolution + private static final int WAIT = 2000; private static final String UNEXPECTED_ARCHIVE_SIZE = "Unexpected archive size"; private static final String UNEXPECTED_FILE_CONTENTS = "Unexpected file contents"; private static final String UNEXPECTED_TREE_CONTENTS = "Unexpected tree contents"; + private static final String UNEXPECTED_LAST_MODIFIED = + "Unexpected lastModified mocked by MockSystemReader, truncated to 1 second"; + private static final String UNEXPECTED_DIFFERENT_HASH = "Unexpected different hash"; private MockFormat format = null; @@ -77,25 +106,20 @@ public class ArchiveCommandTest extends RepositoryTestCase { public void setup() { format = new MockFormat(); ArchiveCommand.registerFormat(format.SUFFIXES.get(0), format); + ArchiveFormats.registerAll(); } @Override @After public void tearDown() { ArchiveCommand.unregisterFormat(format.SUFFIXES.get(0)); + ArchiveFormats.unregisterAll(); } @Test public void archiveHeadAllFiles() throws IOException, GitAPIException { try (Git git = new Git(db)) { - writeTrashFile("file_1.txt", "content_1_1"); - git.add().addFilepattern("file_1.txt").call(); - git.commit().setMessage("create file").call(); - - writeTrashFile("file_1.txt", "content_1_2"); - writeTrashFile("file_2.txt", "content_2_2"); - git.add().addFilepattern(".").call(); - git.commit().setMessage("updated file").call(); + createTestContent(git); git.archive().setOutputStream(new MockOutputStream()) .setFormat(format.SUFFIXES.get(0)) @@ -190,6 +214,157 @@ public void archiveByDirectoryPath() throws GitAPIException, IOException { } } + @Test + public void archiveHeadAllFilesTarTimestamps() throws Exception { + try (Git git = new Git(db)) { + createTestContent(git); + String fmt = "tar"; + File archive = new File(getTemporaryDirectory(), + "archive." + format); + archive(git, archive, fmt); + ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi); + ArchiveInputStream o = new TarArchiveInputStream(bi)) { + assertEntries(o); + } + + Thread.sleep(WAIT); + archive(git, archive, fmt); + assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, + ObjectId.fromRaw(IO.readFully(archive))); + } + } + + @Test + public void archiveHeadAllFilesTgzTimestamps() throws Exception { + try (Git git = new Git(db)) { + createTestContent(git); + String fmt = "tgz"; + File archive = new File(getTemporaryDirectory(), + "archive." + fmt); + archive(git, archive, fmt); + ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi); + InputStream gzi = new GzipCompressorInputStream(bi); + ArchiveInputStream o = new TarArchiveInputStream(gzi)) { + assertEntries(o); + } + + Thread.sleep(WAIT); + archive(git, archive, fmt); + assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, + ObjectId.fromRaw(IO.readFully(archive))); + } + } + + @Test + public void archiveHeadAllFilesTbz2Timestamps() throws Exception { + try (Git git = new Git(db)) { + createTestContent(git); + String fmt = "tbz2"; + File archive = new File(getTemporaryDirectory(), + "archive." + fmt); + archive(git, archive, fmt); + ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi); + InputStream gzi = new BZip2CompressorInputStream(bi); + ArchiveInputStream o = new TarArchiveInputStream(gzi)) { + assertEntries(o); + } + + Thread.sleep(WAIT); + archive(git, archive, fmt); + assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, + ObjectId.fromRaw(IO.readFully(archive))); + } + } + + @Test + public void archiveHeadAllFilesTxzTimestamps() throws Exception { + try (Git git = new Git(db)) { + createTestContent(git); + String fmt = "txz"; + File archive = new File(getTemporaryDirectory(), "archive." + fmt); + archive(git, archive, fmt); + ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi); + InputStream gzi = new XZCompressorInputStream(bi); + ArchiveInputStream o = new TarArchiveInputStream(gzi)) { + assertEntries(o); + } + + Thread.sleep(WAIT); + archive(git, archive, fmt); + assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, + ObjectId.fromRaw(IO.readFully(archive))); + } + } + + @Test + public void archiveHeadAllFilesZipTimestamps() throws Exception { + try (Git git = new Git(db)) { + createTestContent(git); + String fmt = "zip"; + File archive = new File(getTemporaryDirectory(), "archive." + fmt); + archive(git, archive, fmt); + ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi); + ArchiveInputStream o = new ZipArchiveInputStream(bi)) { + assertEntries(o); + } + + Thread.sleep(WAIT); + archive(git, archive, fmt); + assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, + ObjectId.fromRaw(IO.readFully(archive))); + } + } + + private void createTestContent(Git git) throws IOException, GitAPIException, + NoFilepatternException, NoHeadException, NoMessageException, + UnmergedPathsException, ConcurrentRefUpdateException, + WrongRepositoryStateException, AbortedByHookException { + writeTrashFile("file_1.txt", "content_1_1"); + git.add().addFilepattern("file_1.txt").call(); + git.commit().setMessage("create file").call(); + + writeTrashFile("file_1.txt", "content_1_2"); + writeTrashFile("file_2.txt", "content_2_2"); + git.add().addFilepattern(".").call(); + git.commit().setMessage("updated file").call(); + } + + private static void archive(Git git, File archive, String fmt) + throws GitAPIException, + FileNotFoundException, AmbiguousObjectException, + IncorrectObjectTypeException, IOException { + git.archive().setOutputStream(new FileOutputStream(archive)) + .setFormat(fmt) + .setTree(git.getRepository().resolve("HEAD")).call(); + } + + private static void assertEntries(ArchiveInputStream o) throws IOException { + ArchiveEntry e; + int n = 0; + while ((e = o.getNextEntry()) != null) { + n++; + assertEquals(UNEXPECTED_LAST_MODIFIED, + (1250379778668L / 1000L) * 1000L, + e.getLastModifiedDate().getTime()); + } + assertEquals(UNEXPECTED_ARCHIVE_SIZE, 2, n); + } + private static class MockFormat implements ArchiveCommand.Format { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java index 27bb5a90b..3f7306bf3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java @@ -56,13 +56,18 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilterGroup; @@ -375,13 +380,15 @@ private OutputStream writeArchive(Format fmt) { MutableObjectId idBuf = new MutableObjectId(); ObjectReader reader = walk.getObjectReader(); - walk.reset(rw.parseTree(tree)); - if (!paths.isEmpty()) + RevObject o = rw.peel(rw.parseAny(tree)); + walk.reset(getTree(o)); + if (!paths.isEmpty()) { walk.setFilter(PathFilterGroup.createFromStrings(paths)); + } // Put base directory into archive if (pfx.endsWith("/")) { //$NON-NLS-1$ - fmt.putEntry(outa, tree, pfx.replaceAll("[/]+$", "/"), //$NON-NLS-1$ //$NON-NLS-2$ + fmt.putEntry(outa, o, pfx.replaceAll("[/]+$", "/"), //$NON-NLS-1$ //$NON-NLS-2$ FileMode.TREE, null); } @@ -392,17 +399,18 @@ private OutputStream writeArchive(Format fmt) { if (walk.isSubtree()) walk.enterSubtree(); - if (mode == FileMode.GITLINK) + if (mode == FileMode.GITLINK) { // TODO(jrn): Take a callback to recurse // into submodules. mode = FileMode.TREE; + } if (mode == FileMode.TREE) { - fmt.putEntry(outa, tree, name + "/", mode, null); //$NON-NLS-1$ + fmt.putEntry(outa, o, name + "/", mode, null); //$NON-NLS-1$ continue; } walk.getObjectId(idBuf, 0); - fmt.putEntry(outa, tree, name, mode, reader.open(idBuf)); + fmt.putEntry(outa, o, name, mode, reader.open(idBuf)); } return out; } finally { @@ -534,4 +542,19 @@ public ArchiveCommand setPaths(String... paths) { this.paths = Arrays.asList(paths); return this; } + + private RevTree getTree(RevObject o) + throws IncorrectObjectTypeException { + final RevTree t; + if (o instanceof RevCommit) { + t = ((RevCommit) o).getTree(); + } else if (!(o instanceof RevTree)) { + throw new IncorrectObjectTypeException(tree.toObjectId(), + Constants.TYPE_TREE); + } else { + t = (RevTree) o; + } + return t; + } + } From f18b5010fcf750e6949e53dca292db9f2b4dc57d Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Sat, 29 Sep 2018 14:39:56 +0900 Subject: [PATCH 06/47] Deprecate Constants.CHARACTER_ENCODING in favor of StandardCharsets.UTF_8 Change-Id: I621ba174235a6fb56236e54d24bce704bb5afb28 Signed-off-by: David Pursehouse --- .../src/org/eclipse/jgit/archive/TarFormat.java | 7 +++---- .../jgit/http/server/InfoRefsServlet.java | 2 +- .../eclipse/jgit/http/server/ServletUtils.java | 5 +++-- .../http/test/SmartClientSmartServerTest.java | 2 +- .../eclipse/jgit/lfs/lib/LFSPointerTest.java | 4 ++-- .../org/eclipse/jgit/pgm/ProxyConfigTest.java | 4 ++-- .../org/eclipse/jgit/lib/MergeHeadMsgTest.java | 9 ++++++--- .../tst/org/eclipse/jgit/lib/RacyGitTests.java | 3 ++- .../eclipse/jgit/lib/SquashCommitMsgTest.java | 3 ++- .../eclipse/jgit/merge/MergeAlgorithmTest.java | 3 ++- .../jgit/transport/PacketLineOutTest.java | 6 +++--- .../transport/SideBandOutputStreamTest.java | 7 +++---- .../src/org/eclipse/jgit/api/RebaseCommand.java | 9 +++++---- .../org/eclipse/jgit/gitrepo/RepoCommand.java | 8 ++++---- .../src/org/eclipse/jgit/lib/Constants.java | 10 ++++++++-- .../src/org/eclipse/jgit/lib/Repository.java | 3 ++- .../src/org/eclipse/jgit/transport/URIish.java | 17 ++++------------- .../eclipse/jgit/treewalk/FileTreeIterator.java | 4 +++- .../src/org/eclipse/jgit/util/FS.java | 4 +++- .../src/org/eclipse/jgit/util/HttpSupport.java | 4 ++-- 20 files changed, 61 insertions(+), 53 deletions(-) diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TarFormat.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TarFormat.java index 9ed60d941..aee80d830 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TarFormat.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TarFormat.java @@ -42,7 +42,7 @@ */ package org.eclipse.jgit.archive; -import static org.eclipse.jgit.lib.Constants.CHARACTER_ENCODING; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.io.OutputStream; @@ -85,7 +85,7 @@ public ArchiveOutputStream createArchiveOutputStream(OutputStream s) public ArchiveOutputStream createArchiveOutputStream(OutputStream s, Map o) throws IOException { TarArchiveOutputStream out = new TarArchiveOutputStream(s, - CHARACTER_ENCODING); + UTF_8.name()); out.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); out.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); return applyFormatOptions(out, o); @@ -99,8 +99,7 @@ public void putEntry(ArchiveOutputStream out, if (mode == FileMode.SYMLINK) { final TarArchiveEntry entry = new TarArchiveEntry( path, TarConstants.LF_SYMLINK); - entry.setLinkName(new String( - loader.getCachedBytes(100), CHARACTER_ENCODING)); + entry.setLinkName(new String(loader.getCachedBytes(100), UTF_8)); out.putArchiveEntry(entry); out.closeArchiveEntry(); return; diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoRefsServlet.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoRefsServlet.java index 1a9d19245..b084b0db5 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoRefsServlet.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoRefsServlet.java @@ -69,7 +69,7 @@ public void doGet(final HttpServletRequest req, // Assume a dumb client and send back the dumb client // version of the info/refs file. rsp.setContentType(HttpSupport.TEXT_PLAIN); - rsp.setCharacterEncoding(Constants.CHARACTER_ENCODING); + rsp.setCharacterEncoding(UTF_8.name()); final Repository db = getRepository(req); try (OutputStreamWriter out = new OutputStreamWriter( diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java index 9601c8caf..b6d73b559 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ServletUtils.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.http.server; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; @@ -191,9 +192,9 @@ public static void consumeRequestBody(InputStream in) { public static void sendPlainText(final String content, final HttpServletRequest req, final HttpServletResponse rsp) throws IOException { - final byte[] raw = content.getBytes(Constants.CHARACTER_ENCODING); + final byte[] raw = content.getBytes(UTF_8); rsp.setContentType(TEXT_PLAIN); - rsp.setCharacterEncoding(Constants.CHARACTER_ENCODING); + rsp.setCharacterEncoding(UTF_8.name()); send(raw, req, rsp); } diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java index 2b4a2511e..b26324d4f 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java @@ -298,7 +298,7 @@ public void doFilter(ServletRequest request, throws IOException, ServletException { final HttpServletResponse r = (HttpServletResponse) response; r.setContentType("text/plain"); - r.setCharacterEncoding(Constants.CHARACTER_ENCODING); + r.setCharacterEncoding(UTF_8.name()); try (PrintWriter w = r.getWriter()) { w.print("OK"); } diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java index a1283ddf4..146a25ed3 100644 --- a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.lfs.lib; -import static org.eclipse.jgit.lib.Constants.CHARACTER_ENCODING; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream; @@ -66,7 +66,7 @@ public void testEncoding() throws IOException { assertEquals( "version https://git-lfs.github.com/spec/v1\noid sha256:" + s + "\nsize 4\n", - baos.toString(CHARACTER_ENCODING)); + baos.toString(UTF_8.name())); } } } diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ProxyConfigTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ProxyConfigTest.java index 40a223d92..42530f3a3 100644 --- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ProxyConfigTest.java +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/ProxyConfigTest.java @@ -37,6 +37,7 @@ */ package org.eclipse.jgit.pgm; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream; @@ -47,7 +48,6 @@ import java.util.List; import java.util.Map; -import org.eclipse.jgit.lib.Constants; import org.junit.Before; import org.junit.Test; @@ -204,7 +204,7 @@ private static String getOutput(Process p) while ((length = inputStream.read(buffer)) != -1) { result.write(buffer, 0, length); } - return result.toString(Constants.CHARACTER_ENCODING); + return result.toString(UTF_8.name()); } } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/MergeHeadMsgTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/MergeHeadMsgTest.java index 347883f84..abf7d5681 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/MergeHeadMsgTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/MergeHeadMsgTest.java @@ -42,6 +42,7 @@ */ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -72,7 +73,9 @@ public void testReadWriteMergeHeads() throws IOException { // same test again, this time with lower-level io try (FileOutputStream fos = new FileOutputStream( new File(db.getDirectory(), "MERGE_HEAD"));) { - fos.write("0000000000000000000000000000000000000000\n1c6db447abdbb291b25f07be38ea0b1bf94947c5\n".getBytes(Constants.CHARACTER_ENCODING)); + fos.write( + "0000000000000000000000000000000000000000\n1c6db447abdbb291b25f07be38ea0b1bf94947c5\n" + .getBytes(UTF_8)); } assertEquals(db.readMergeHeads().size(), 2); assertEquals(db.readMergeHeads().get(0), ObjectId.zeroId()); @@ -82,7 +85,7 @@ public void testReadWriteMergeHeads() throws IOException { assertEquals(db.readMergeHeads(), null); try (FileOutputStream fos = new FileOutputStream( new File(db.getDirectory(), "MERGE_HEAD"))) { - fos.write(sampleId.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(sampleId.getBytes(UTF_8)); } assertEquals(db.readMergeHeads().size(), 1); assertEquals(db.readMergeHeads().get(0), ObjectId.fromString(sampleId)); @@ -100,7 +103,7 @@ public void testReadWriteMergeMsg() throws IOException { assertFalse(new File(db.getDirectory(), "MERGE_MSG").exists()); try (FileOutputStream fos = new FileOutputStream( new File(db.getDirectory(), Constants.MERGE_MSG))) { - fos.write(mergeMsg.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(mergeMsg.getBytes(UTF_8)); } assertEquals(db.readMergeCommitMsg(), mergeMsg); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java index 3542dfad2..bb24994ee 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.lib; import static java.lang.Long.valueOf; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @@ -181,7 +182,7 @@ public void testRacyGitDetection() throws Exception { private File addToWorkDir(String path, String content) throws IOException { File f = new File(db.getWorkTree(), path); try (FileOutputStream fos = new FileOutputStream(f)) { - fos.write(content.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(content.getBytes(UTF_8)); return f; } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/SquashCommitMsgTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/SquashCommitMsgTest.java index 203c00e28..f58ab0607 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/SquashCommitMsgTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/SquashCommitMsgTest.java @@ -42,6 +42,7 @@ */ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -68,7 +69,7 @@ public void testReadWriteMergeMsg() throws IOException { assertFalse(new File(db.getDirectory(), Constants.SQUASH_MSG).exists()); try (FileOutputStream fos = new FileOutputStream( new File(db.getDirectory(), Constants.SQUASH_MSG))) { - fos.write(squashMsg.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(squashMsg.getBytes(UTF_8)); } assertEquals(db.readSquashCommitMsg(), squashMsg); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmTest.java index 5af62b670..b13180831 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmTest.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.merge; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream; @@ -302,7 +303,7 @@ private String merge(String commonBase, String ours, String theirs) throws IOExc T(commonBase), T(ours), T(theirs)); ByteArrayOutputStream bo=new ByteArrayOutputStream(50); fmt.formatMerge(bo, r, "B", "O", "T", Constants.CHARACTER_ENCODING); - return new String(bo.toByteArray(), Constants.CHARACTER_ENCODING); + return new String(bo.toByteArray(), UTF_8); } public String t(String text) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineOutTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineOutTest.java index 391a701b8..ad8ae4253 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineOutTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineOutTest.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -50,7 +51,6 @@ import java.io.IOException; import java.io.OutputStream; -import org.eclipse.jgit.lib.Constants; import org.junit.Before; import org.junit.Test; @@ -173,8 +173,8 @@ public void flush() throws IOException { assertEquals(1, flushCnt[0]); } - private void assertBuffer(String exp) throws IOException { + private void assertBuffer(String exp) { assertEquals(exp, new String(rawOut.toByteArray(), - Constants.CHARACTER_ENCODING)); + UTF_8)); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/SideBandOutputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/SideBandOutputStreamTest.java index 4d3e16224..b6cf3564c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/SideBandOutputStreamTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/SideBandOutputStreamTest.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.transport; import static java.lang.Integer.valueOf; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.transport.SideBandOutputStream.CH_DATA; import static org.eclipse.jgit.transport.SideBandOutputStream.CH_ERROR; import static org.eclipse.jgit.transport.SideBandOutputStream.CH_PROGRESS; @@ -59,7 +60,6 @@ import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.Constants; import org.junit.Before; import org.junit.Test; @@ -259,8 +259,7 @@ public void testConstructor_RejectsBadBufferSize() throws Exception { } } - private void assertBuffer(String exp) throws IOException { - assertEquals(exp, new String(rawOut.toByteArray(), - Constants.CHARACTER_ENCODING)); + private void assertBuffer(String exp) { + assertEquals(exp, new String(rawOut.toByteArray(), UTF_8)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 1783c4193..9653c365b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -43,6 +43,8 @@ */ package org.eclipse.jgit.api; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; @@ -1015,8 +1017,7 @@ private RebaseResult stop(RevCommit commitToPick, RebaseResult.Status status) df.setRepository(repo); df.format(commitToPick.getParent(0), commitToPick); } - rebaseState.createFile(PATCH, new String(bos.toByteArray(), - Constants.CHARACTER_ENCODING)); + rebaseState.createFile(PATCH, new String(bos.toByteArray(), UTF_8)); rebaseState.createFile(STOPPED_SHA, repo.newObjectReader() .abbreviate( @@ -1733,7 +1734,7 @@ private static void createFile(File parentDir, String name, throws IOException { File file = new File(parentDir, name); try (FileOutputStream fos = new FileOutputStream(file)) { - fos.write(content.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(content.getBytes(UTF_8)); fos.write('\n'); } } @@ -1741,7 +1742,7 @@ private static void createFile(File parentDir, String name, private static void appendToFile(File file, String content) throws IOException { try (FileOutputStream fos = new FileOutputStream(file, true)) { - fos.write(content.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(content.getBytes(UTF_8)); fos.write('\n'); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 45a239da0..5a73cdc06 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -42,6 +42,7 @@ */ package org.eclipse.jgit.gitrepo; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME; import static org.eclipse.jgit.lib.Constants.R_REMOTES; @@ -606,8 +607,7 @@ public RevCommit call() throws GitAPIException { } objectId = inserter.insert(Constants.OBJ_BLOB, - link.getBytes( - Constants.CHARACTER_ENCODING)); + link.getBytes(UTF_8)); dcEntry = new DirCacheEntry(linkfile.dest); dcEntry.setObjectId(objectId); dcEntry.setFileMode(FileMode.SYMLINK); @@ -620,7 +620,7 @@ public RevCommit call() throws GitAPIException { // create a new DirCacheEntry for .gitmodules file. final DirCacheEntry dcEntry = new DirCacheEntry(Constants.DOT_GIT_MODULES); ObjectId objectId = inserter.insert(Constants.OBJ_BLOB, - content.getBytes(Constants.CHARACTER_ENCODING)); + content.getBytes(UTF_8)); dcEntry.setObjectId(objectId); dcEntry.setFileMode(FileMode.REGULAR_FILE); builder.add(dcEntry); @@ -629,7 +629,7 @@ public RevCommit call() throws GitAPIException { // create a new DirCacheEntry for .gitattributes file. final DirCacheEntry dcEntryAttr = new DirCacheEntry(Constants.DOT_GIT_ATTRIBUTES); ObjectId attrId = inserter.insert(Constants.OBJ_BLOB, - attributes.toString().getBytes(Constants.CHARACTER_ENCODING)); + attributes.toString().getBytes(UTF_8)); dcEntryAttr.setObjectId(attrId); dcEntryAttr.setFileMode(FileMode.REGULAR_FILE); builder.add(dcEntryAttr); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index ed0055416..4c5519696 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -232,11 +232,17 @@ public final class Constants { * * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly * instead. - **/ + */ @Deprecated public static final Charset CHARSET; - /** Native character encoding for commit messages, file names... */ + /** + * Native character encoding for commit messages, file names... + * + * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly + * instead. + */ + @Deprecated public static final String CHARACTER_ENCODING; /** Default main branch name */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index d73c05e24..2a2699f90 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -49,6 +49,7 @@ package org.eclipse.jgit.lib; import static org.eclipse.jgit.lib.Constants.LOCK_SUFFIX; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.BufferedOutputStream; import java.io.File; @@ -1965,7 +1966,7 @@ private String readCommitMsgFile(String msgFilename) throws IOException { private void writeCommitMsg(File msgFile, String msg) throws IOException { if (msg != null) { try (FileOutputStream fos = new FileOutputStream(msgFile)) { - fos.write(msg.getBytes(Constants.CHARACTER_ENCODING)); + fos.write(msg.getBytes(UTF_8)); } } else { FileUtils.delete(msgFile, FileUtils.SKIP_MISSING); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java index 026fd819c..70fb1f0e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java @@ -48,10 +48,11 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.Serializable; -import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; import java.net.URL; import java.util.BitSet; @@ -282,12 +283,7 @@ private static String unescape(String s) throws URISyntaxException { if (s.indexOf('%') < 0) return s; - byte[] bytes; - try { - bytes = s.getBytes(Constants.CHARACTER_ENCODING); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // can't happen - } + byte[] bytes = s.getBytes(UTF_8); byte[] os = new byte[bytes.length]; int j = 0; @@ -335,12 +331,7 @@ private static String escape(String s, boolean escapeReservedChars, if (s == null) return null; ByteArrayOutputStream os = new ByteArrayOutputStream(s.length()); - byte[] bytes; - try { - bytes = s.getBytes(Constants.CHARACTER_ENCODING); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // cannot happen - } + byte[] bytes = s.getBytes(UTF_8); for (int i = 0; i < bytes.length; ++i) { int b = bytes[i] & 0xFF; if (b <= 32 || (encodeNonAscii && b > 127) || b == '%' diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java index 24b9ac086..3d25c2314 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java @@ -46,6 +46,8 @@ package org.eclipse.jgit.treewalk; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -412,7 +414,7 @@ public long getLastModified() { public InputStream openInputStream() throws IOException { if (attributes.isSymbolicLink()) { return new ByteArrayInputStream(fs.readSymLink(getFile()) - .getBytes(Constants.CHARACTER_ENCODING)); + .getBytes(UTF_8)); } else { return new FileInputStream(getFile()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 180123e09..e559d2167 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -43,6 +43,8 @@ package org.eclipse.jgit.util; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.Closeable; @@ -1286,7 +1288,7 @@ public int runProcess(ProcessBuilder processBuilder, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws IOException, InterruptedException { InputStream in = (stdinArgs == null) ? null : new ByteArrayInputStream( - stdinArgs.getBytes(Constants.CHARACTER_ENCODING)); + stdinArgs.getBytes(UTF_8)); return runProcess(processBuilder, outRedirect, errRedirect, in); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java index 6f92b3785..9190a5915 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java @@ -44,7 +44,7 @@ package org.eclipse.jgit.util; -import static org.eclipse.jgit.lib.Constants.CHARACTER_ENCODING; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -181,7 +181,7 @@ public static void encode(StringBuilder urlstr, String key) { if (key == null || key.length() == 0) return; try { - urlstr.append(URLEncoder.encode(key, CHARACTER_ENCODING)); + urlstr.append(URLEncoder.encode(key, UTF_8.name())); } catch (UnsupportedEncodingException e) { throw new RuntimeException(JGitText.get().couldNotURLEncodeToUTF8, e); } From df637928d2ef4b9ee06af7e37344c7848f870ce4 Mon Sep 17 00:00:00 2001 From: Masaya Suzuki Date: Sun, 23 Dec 2018 19:31:10 -0800 Subject: [PATCH 07/47] Change RacyGitTests to create a racy git situation in a stable way By using File#setLastModified, we can create a racy git situation stably. Tested with --runs_per_test=100 Bug: 526111 Change-Id: I60b3632d353e19f335668325aa603640be423f58 Signed-off-by: Masaya Suzuki --- .../org/eclipse/jgit/lib/RacyGitTests.java | 53 +++++++++---------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java index bb24994ee..11100b63c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java @@ -46,7 +46,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.FileOutputStream; @@ -63,8 +62,8 @@ public class RacyGitTests extends RepositoryTestCase { @Test - public void testIterator() throws IllegalStateException, IOException, - InterruptedException { + public void testIterator() + throws IllegalStateException, IOException, InterruptedException { TreeSet modTimes = new TreeSet<>(); File lastFile = null; for (int i = 0; i < 10; i++) { @@ -128,9 +127,6 @@ public void testIterator() throws IllegalStateException, IOException, @Test public void testRacyGitDetection() throws Exception { - TreeSet modTimes = new TreeSet<>(); - File lastFile; - // Reset to force creation of index file try (Git git = new Git(db)) { git.reset().call(); @@ -138,45 +134,44 @@ public void testRacyGitDetection() throws Exception { // wait to ensure that modtimes of the file doesn't match last index // file modtime - modTimes.add(valueOf(fsTick(db.getIndexFile()))); + fsTick(db.getIndexFile()); // create two files - addToWorkDir("a", "a"); - lastFile = addToWorkDir("b", "b"); + File a = addToWorkDir("a", "a"); + File b = addToWorkDir("b", "b"); + assertTrue(a.setLastModified(b.lastModified())); + assertTrue(b.setLastModified(b.lastModified())); // wait to ensure that file-modTimes and therefore index entry modTime // doesn't match the modtime of index-file after next persistance - modTimes.add(valueOf(fsTick(lastFile))); + fsTick(b); // now add both files to the index. No racy git expected - resetIndex(new FileTreeIteratorWithTimeControl(db, modTimes)); + resetIndex(new FileTreeIterator(db)); assertEquals( - "[a, mode:100644, time:t0, length:1, content:a]" + - "[b, mode:100644, time:t0, length:1, content:b]", + "[a, mode:100644, time:t0, length:1, content:a]" + + "[b, mode:100644, time:t0, length:1, content:b]", indexState(SMUDGE | MOD_TIME | LENGTH | CONTENT)); - // Remember the last modTime of index file. All modifications times of - // further modification are translated to this value so it looks that - // files have been modified in the same time slot as the index file - long indexMod = db.getIndexFile().lastModified(); - modTimes.add(Long.valueOf(indexMod)); + // wait to ensure the file 'a' is updated at t1. + fsTick(db.getIndexFile()); - // modify one file - long aMod = addToWorkDir("a", "a2").lastModified(); - assumeTrue(aMod == indexMod); - - // now update the index the index. 'a' has to be racily clean -- because - // it's modification time is exactly the same as the previous index file - // mod time. - resetIndex(new FileTreeIteratorWithTimeControl(db, modTimes)); + // Create a racy git situation. This is a situation that the index is + // updated and then a file is modified within a second. By changing the + // index file artificially, we create a fake racy situation. + File updatedA = addToWorkDir("a", "a2"); + assertTrue(updatedA.setLastModified(updatedA.lastModified() + 100)); + resetIndex(new FileTreeIterator(db)); + assertTrue(db.getIndexFile() + .setLastModified(updatedA.lastModified() + 90)); db.readDirCache(); // although racily clean a should not be reported as being dirty assertEquals( - "[a, mode:100644, time:t1, smudged, length:0, content:a2]" + - "[b, mode:100644, time:t0, length:1, content:b]", - indexState(SMUDGE|MOD_TIME|LENGTH|CONTENT)); + "[a, mode:100644, time:t1, smudged, length:0, content:a2]" + + "[b, mode:100644, time:t0, length:1, content:b]", + indexState(SMUDGE | MOD_TIME | LENGTH | CONTENT)); } private File addToWorkDir(String path, String content) throws IOException { From 1159f9dd7c80a53c2509cd75d997a6afed37f9a6 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 18 Jun 2019 11:32:59 +0200 Subject: [PATCH 08/47] Fix RacyGitTests#testRacyGitDetection This test case assumed file system timestamp resolution of 1 second. On filesystems with a finer resolution this test fails since the index entry is only smudged if the file index entry's lastModified and the lastModified of the git index itself are within the same filesystem timer tick. Fix this by ensuring that these timestamps are identical which should work for any filesystem timer resolution. Bug: 548188 Change-Id: Id84d59e1cfeb48fa008f8f27f2f892c4f73985de Signed-off-by: Matthias Sohn Signed-off-by: Christian Halstrick --- .../org/eclipse/jgit/lib/RacyGitTests.java | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java index 11100b63c..c1b6cb259 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java @@ -45,6 +45,7 @@ import static java.lang.Long.valueOf; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.File; @@ -53,10 +54,12 @@ import java.util.TreeSet; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.treewalk.FileTreeIterator; import org.eclipse.jgit.treewalk.FileTreeIteratorWithTimeControl; import org.eclipse.jgit.treewalk.NameConflictTreeWalk; +import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.util.FileUtils; import org.junit.Test; @@ -137,8 +140,8 @@ public void testRacyGitDetection() throws Exception { fsTick(db.getIndexFile()); // create two files - File a = addToWorkDir("a", "a"); - File b = addToWorkDir("b", "b"); + File a = writeToWorkDir("a", "a"); + File b = writeToWorkDir("b", "b"); assertTrue(a.setLastModified(b.lastModified())); assertTrue(b.setLastModified(b.lastModified())); @@ -158,23 +161,35 @@ public void testRacyGitDetection() throws Exception { fsTick(db.getIndexFile()); // Create a racy git situation. This is a situation that the index is - // updated and then a file is modified within a second. By changing the - // index file artificially, we create a fake racy situation. - File updatedA = addToWorkDir("a", "a2"); - assertTrue(updatedA.setLastModified(updatedA.lastModified() + 100)); + // updated and then a file is modified within the same tick of the + // filesystem timestamp resolution. By changing the index file + // artificially, we create a fake racy situation. + File updatedA = writeToWorkDir("a", "a2"); + long newLastModified = updatedA.lastModified() + 100; + assertTrue(updatedA.setLastModified(newLastModified)); resetIndex(new FileTreeIterator(db)); - assertTrue(db.getIndexFile() - .setLastModified(updatedA.lastModified() + 90)); + assertTrue(db.getIndexFile().setLastModified(newLastModified)); - db.readDirCache(); - // although racily clean a should not be reported as being dirty + DirCache dc = db.readDirCache(); + // check index state: although racily clean a should not be reported as + // being dirty since we forcefully reset the index to match the working + // tree assertEquals( "[a, mode:100644, time:t1, smudged, length:0, content:a2]" + "[b, mode:100644, time:t0, length:1, content:b]", indexState(SMUDGE | MOD_TIME | LENGTH | CONTENT)); + + // compare state of files in working tree with index to check that + // FileTreeIterator.isModified() works as expected + FileTreeIterator f = new FileTreeIterator(db.getWorkTree(), db.getFS(), + db.getConfig().get(WorkingTreeOptions.KEY)); + assertTrue(f.findFile("a")); + try (ObjectReader reader = db.newObjectReader()) { + assertFalse(f.isModified(dc.getEntry("a"), false, reader)); + } } - private File addToWorkDir(String path, String content) throws IOException { + private File writeToWorkDir(String path, String content) throws IOException { File f = new File(db.getWorkTree(), path); try (FileOutputStream fos = new FileOutputStream(f)) { fos.write(content.getBytes(UTF_8)); From 850b9d7540025a974870bc75a2412ee91469a198 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 21 Jun 2019 11:00:05 +0200 Subject: [PATCH 09/47] Timeout measuring file timestamp resolution after 2 seconds It was reported that measuring file timestamp resolution may hang indefinitely on nfs. Hence timeout this measurement at the known worst filesystem timestamp resolution (FAT) of 2 seconds. Bug: 548188 Change-Id: I17004b0aa49d5b0e76360a008af3adb911b289c0 Signed-off-by: Matthias Sohn --- .../eclipse/jgit/internal/JGitText.properties | 1 + .../src/org/eclipse/jgit/internal/JGitText.java | 1 + .../src/org/eclipse/jgit/util/FS.java | 17 +++++++++++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index b7805b19e..0a2f7029e 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -680,6 +680,7 @@ theFactoryMustNotBeNull=The factory must not be null threadInterruptedWhileRunning="Current thread interrupted while running {0}" timeIsUncertain=Time is uncertain timerAlreadyTerminated=Timer already terminated +timeoutMeasureFsTimestampResolution=measuring filesystem timestamp resolution for ''{0}'' timed out, fall back to resolution of 2 seconds tooManyCommands=Too many commands tooManyFilters=Too many "filter" lines in request tooManyIncludeRecursions=Too many recursions; circular includes in config file(s)? diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index da0ba4a77..bcd6d5ce2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -737,6 +737,7 @@ public static JGitText get() { /***/ public String tagAlreadyExists; /***/ public String tagNameInvalid; /***/ public String tagOnRepoWithoutHEADCurrentlyNotSupported; + /***/ public String timeoutMeasureFsTimestampResolution; /***/ public String transactionAborted; /***/ public String theFactoryMustNotBeNull; /***/ public String threadInterruptedWhileRunning; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index e559d2167..e7db6cee7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -208,7 +208,7 @@ static Duration getFsTimestampResolution(Path file) { FileStore s = Files.getFileStore(dir); FileStoreAttributeCache c = attributeCache.get(s); if (c == null) { - c = new FileStoreAttributeCache(dir); + c = new FileStoreAttributeCache(s, dir); attributeCache.put(s, c); if (LOG.isDebugEnabled()) { LOG.debug(c.toString()); @@ -228,16 +228,24 @@ Duration getFsTimestampResolution() { return fsTimestampResolution; } - private FileStoreAttributeCache(Path dir) + private FileStoreAttributeCache(FileStore s, Path dir) throws IOException, InterruptedException { Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ Files.createFile(probe); try { + long start = System.nanoTime(); FileTime startTime = Files.getLastModifiedTime(probe); FileTime actTime = startTime; long sleepTime = 512; while (actTime.compareTo(startTime) <= 0) { TimeUnit.NANOSECONDS.sleep(sleepTime); + if (timeout(start)) { + LOG.warn(MessageFormat.format(JGitText + .get().timeoutMeasureFsTimestampResolution, + s.toString())); + fsTimestampResolution = FALLBACK_TIMESTAMP_RESOLUTION; + return; + } FileUtils.touch(probe); actTime = Files.getLastModifiedTime(probe); // limit sleep time to max. 100ms @@ -254,6 +262,11 @@ private FileStoreAttributeCache(Path dir) } } + private static boolean timeout(long start) { + return System.nanoTime() - start >= FALLBACK_TIMESTAMP_RESOLUTION + .toNanos(); + } + @SuppressWarnings("nls") @Override public String toString() { From 84e6c24e58f9d5dbbeef9fd299e5c5598e7a48e7 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Tue, 9 Jul 2019 14:49:30 +0200 Subject: [PATCH 10/47] FileSnapshot#equals: consider UNKNOWN_SIZE Add a unittest. In commit I5485db55 ("Fix FileSnapshot's consideration of file size"), the special casing of UNKNOWN_SIZE was forgotten. This change, together with I493f3b57b ("Measure file timestamp resolution used in FileSnapshot") introduced a regression that would occasionally surface in Gerrit integration tests marked UseLocalDisk, with the symptom that creating the Admin user in NoteDb failed with a LOCK_FAILURE. Signed-off-by: Han-Wen Nienhuys Change-Id: I7ffd972581f815c144f810481103c7985af5feb0 --- .../jgit/internal/storage/file/FileSnapshotTest.java | 11 +++++++++++ .../jgit/internal/storage/file/FileSnapshot.java | 5 +++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 5ebdeb6e8..6e458fbbf 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -180,6 +180,17 @@ public void testFileSizeChanged() throws Exception { assertTrue(save.wasSizeChanged()); } + @Test + public void fileSnapshotEquals() throws Exception { + // 0 sized FileSnapshot. + FileSnapshot fs1 = FileSnapshot.MISSING_FILE; + // UNKNOWN_SIZE FileSnapshot. + FileSnapshot fs2 = FileSnapshot.save(fs1.lastModified()); + + assertTrue(fs1.equals(fs2)); + assertTrue(fs2.equals(fs1)); + } + private File createFile(String string) throws IOException { trash.mkdirs(); File f = File.createTempFile(string, "tdat", trash); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 1de313500..0019c5f09 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -144,7 +144,7 @@ private static Object getFileKey(BasicFileAttributes fileAttributes) { */ public static FileSnapshot save(long modified) { final long read = System.currentTimeMillis(); - return new FileSnapshot(read, modified, -1, Duration.ZERO, + return new FileSnapshot(read, modified, UNKNOWN_SIZE, Duration.ZERO, MISSING_FILEKEY); } @@ -318,7 +318,8 @@ public void waitUntilNotRacy() throws InterruptedException { * @return true if the two snapshots share the same information. */ public boolean equals(FileSnapshot other) { - return lastModified == other.lastModified && size == other.size + boolean sizeEq = size == UNKNOWN_SIZE || other.size == UNKNOWN_SIZE || size == other.size; + return lastModified == other.lastModified && sizeEq && Objects.equals(fileKey, other.fileKey); } From a024759cf5bf1cd6b9beb4f790d484943761a7e1 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 11 Jul 2019 10:00:23 +0200 Subject: [PATCH 11/47] Delete unused FileTreeIteratorWithTimeControl The only usage of this test iterator was removed in df637928d. Hence delete this iterator and associated test. Change-Id: I47710133ec3edc675c21db210960c024982668c6 Signed-off-by: Matthias Sohn --- .../org/eclipse/jgit/lib/RacyGitTests.java | 68 ----------- .../FileTreeIteratorWithTimeControl.java | 109 ------------------ 2 files changed, 177 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorWithTimeControl.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java index c1b6cb259..d044e65cf 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java @@ -42,7 +42,6 @@ */ package org.eclipse.jgit.lib; -import static java.lang.Long.valueOf; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -51,82 +50,15 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.TreeSet; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.treewalk.FileTreeIterator; -import org.eclipse.jgit.treewalk.FileTreeIteratorWithTimeControl; -import org.eclipse.jgit.treewalk.NameConflictTreeWalk; import org.eclipse.jgit.treewalk.WorkingTreeOptions; -import org.eclipse.jgit.util.FileUtils; import org.junit.Test; public class RacyGitTests extends RepositoryTestCase { - @Test - public void testIterator() - throws IllegalStateException, IOException, InterruptedException { - TreeSet modTimes = new TreeSet<>(); - File lastFile = null; - for (int i = 0; i < 10; i++) { - lastFile = new File(db.getWorkTree(), "0." + i); - FileUtils.createNewFile(lastFile); - if (i == 5) - fsTick(lastFile); - } - modTimes.add(valueOf(fsTick(lastFile))); - for (int i = 0; i < 10; i++) { - lastFile = new File(db.getWorkTree(), "1." + i); - FileUtils.createNewFile(lastFile); - } - modTimes.add(valueOf(fsTick(lastFile))); - for (int i = 0; i < 10; i++) { - lastFile = new File(db.getWorkTree(), "2." + i); - FileUtils.createNewFile(lastFile); - if (i % 4 == 0) - fsTick(lastFile); - } - FileTreeIteratorWithTimeControl fileIt = new FileTreeIteratorWithTimeControl( - db, modTimes); - try (NameConflictTreeWalk tw = new NameConflictTreeWalk(db)) { - tw.addTree(fileIt); - tw.setRecursive(true); - FileTreeIterator t; - long t0 = 0; - for (int i = 0; i < 10; i++) { - assertTrue(tw.next()); - t = tw.getTree(0, FileTreeIterator.class); - if (i == 0) { - t0 = t.getEntryLastModified(); - } else { - assertEquals(t0, t.getEntryLastModified()); - } - } - long t1 = 0; - for (int i = 0; i < 10; i++) { - assertTrue(tw.next()); - t = tw.getTree(0, FileTreeIterator.class); - if (i == 0) { - t1 = t.getEntryLastModified(); - assertTrue(t1 > t0); - } else { - assertEquals(t1, t.getEntryLastModified()); - } - } - long t2 = 0; - for (int i = 0; i < 10; i++) { - assertTrue(tw.next()); - t = tw.getTree(0, FileTreeIterator.class); - if (i == 0) { - t2 = t.getEntryLastModified(); - assertTrue(t2 > t1); - } else { - assertEquals(t2, t.getEntryLastModified()); - } - } - } - } @Test public void testRacyGitDetection() throws Exception { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorWithTimeControl.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorWithTimeControl.java deleted file mode 100644 index fc79d4586..000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorWithTimeControl.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2010, Christian Halstrick - * and other copyright owners as documented in the project's IP log. - * - * This program and the accompanying materials are made available - * under the terms of the Eclipse Distribution License v1.0 which - * accompanies this distribution, is reproduced below, and is - * available at http://www.eclipse.org/org/documents/edl-v10.php - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the - * names of its contributors may be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.eclipse.jgit.treewalk; - -import java.io.File; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.util.FS; - -/** - * A {@link FileTreeIterator} used in tests which allows to specify explicitly - * what will be returned by {@link #getEntryLastModified()}. This allows to - * write tests where certain files have to have the same modification time. - *

- * This iterator is configured by a list of strictly increasing long values - * t(0), t(1), ..., t(n). For each file with a modification between t(x) and - * t(x+1) [ t(x) <= time < t(x+1) ] this iterator will report t(x). For - * files with a modification time smaller t(0) a modification time of 0 is - * returned. For files with a modification time greater or equal t(n) t(n) will - * be returned. - *

- * This class was written especially to test racy-git problems - */ -public class FileTreeIteratorWithTimeControl extends FileTreeIterator { - private TreeSet modTimes; - - public FileTreeIteratorWithTimeControl(FileTreeIterator p, Repository repo, - TreeSet modTimes) { - super(p, repo.getWorkTree(), repo.getFS()); - this.modTimes = modTimes; - } - - public FileTreeIteratorWithTimeControl(FileTreeIterator p, File f, FS fs, - TreeSet modTimes) { - super(p, f, fs); - this.modTimes = modTimes; - } - - public FileTreeIteratorWithTimeControl(Repository repo, - TreeSet modTimes) { - super(repo); - this.modTimes = modTimes; - } - - public FileTreeIteratorWithTimeControl(File f, FS fs, - TreeSet modTimes) { - super(f, fs, new Config().get(WorkingTreeOptions.KEY)); - this.modTimes = modTimes; - } - - @Override - public AbstractTreeIterator createSubtreeIterator(ObjectReader reader) { - return new FileTreeIteratorWithTimeControl(this, - ((FileEntry) current()).getFile(), fs, modTimes); - } - - @Override - public long getEntryLastModified() { - if (modTimes == null) - return 0; - Long cutOff = Long.valueOf(super.getEntryLastModified() + 1); - SortedSet head = modTimes.headSet(cutOff); - return head.isEmpty() ? 0 : head.last().longValue(); - } -} From a950eac23b94cb5cce5f75b1a7eb9a436950d3ee Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 21 Jun 2019 13:37:32 +0200 Subject: [PATCH 12/47] Optionally measure filesystem timestamp resolution asynchronously In order to avoid blocking on the main thread during measurement interactive applications like EGit may want to measure the filesystem timestamp resolution asynchronously. In order to enable measurement in the background call FileStoreAttributeCache.setAsyncfileStoreAttrCache(true) before the first access to cached FileStore attributes. Bug: 548188 Change-Id: I8c9a2dbfc3f1d33441edea18b90e36b1dc0156c7 Signed-off-by: Matthias Sohn --- org.eclipse.jgit/.settings/.api_filters | 6 + .../src/org/eclipse/jgit/util/FS.java | 197 +++++++++++++----- 2 files changed, 148 insertions(+), 55 deletions(-) diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index a8404dd59..d4c40788d 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -115,6 +115,12 @@ + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index e7db6cee7..2b4e5c78d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -55,7 +55,6 @@ import java.io.OutputStream; import java.io.PrintStream; import java.nio.charset.Charset; -import java.nio.file.AccessDeniedException; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; @@ -71,12 +70,17 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.eclipse.jgit.annotations.NonNull; @@ -197,74 +201,143 @@ private static final class FileStoreAttributeCache { private static final Map attributeCache = new ConcurrentHashMap<>(); - static Duration getFsTimestampResolution(Path file) { + private static AtomicBoolean background = new AtomicBoolean(); + + private static Map locks = new ConcurrentHashMap<>(); + + private static void setBackground(boolean async) { + background.set(async); + } + + private static Duration getFsTimestampResolution(Path file) { + Path dir = Files.isDirectory(file) ? file : file.getParent(); + FileStore s; try { - Path dir = Files.isDirectory(file) ? file : file.getParent(); - if (!dir.toFile().canWrite()) { - // can not determine FileStore of an unborn directory or in - // a read-only directory + if (Files.exists(dir)) { + s = Files.getFileStore(dir); + FileStoreAttributeCache c = attributeCache.get(s); + if (c != null) { + return c.getFsTimestampResolution(); + } + if (!Files.isWritable(dir)) { + // cannot measure resolution in a read-only directory + return FALLBACK_TIMESTAMP_RESOLUTION; + } + } else { + // cannot determine FileStore of an unborn directory return FALLBACK_TIMESTAMP_RESOLUTION; } - FileStore s = Files.getFileStore(dir); - FileStoreAttributeCache c = attributeCache.get(s); - if (c == null) { - c = new FileStoreAttributeCache(s, dir); - attributeCache.put(s, c); - if (LOG.isDebugEnabled()) { - LOG.debug(c.toString()); + CompletableFuture> f = CompletableFuture + .supplyAsync(() -> { + Lock lock = locks.computeIfAbsent(s, + l -> new ReentrantLock()); + if (!lock.tryLock()) { + return Optional.empty(); + } + Optional resolution; + try { + // Some earlier future might have set the value + // and removed itself since we checked for the + // value above. Hence check cache again. + FileStoreAttributeCache c = attributeCache + .get(s); + if (c != null) { + return Optional + .of(c.getFsTimestampResolution()); + } + resolution = measureFsTimestampResolution(s, + dir); + if (resolution.isPresent()) { + FileStoreAttributeCache cache = new FileStoreAttributeCache( + resolution.get()); + attributeCache.put(s, cache); + if (LOG.isDebugEnabled()) { + LOG.debug(cache.toString()); + } + } + } finally { + lock.unlock(); + locks.remove(s); + } + return resolution; + }); + // even if measuring in background wait a little - if the result + // arrives, it's better than returning the large fallback + Optional d = f.get(background.get() ? 50 : 2000, + TimeUnit.MILLISECONDS); + if (d.isPresent()) { + return d.get(); + } + // return fallback until measurement is finished + } catch (IOException | InterruptedException + | ExecutionException e) { + LOG.error(e.getMessage(), e); + } catch (TimeoutException e) { + // use fallback + } + return FALLBACK_TIMESTAMP_RESOLUTION; + } + + private static Optional measureFsTimestampResolution( + FileStore s, Path dir) { + Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ + try { + Files.createFile(probe); + long wait = 512; + long start = System.nanoTime(); + FileTime t1 = Files.getLastModifiedTime(probe); + FileTime t2 = t1; + while (t2.compareTo(t1) <= 0) { + TimeUnit.NANOSECONDS.sleep(wait); + checkTimeout(s, start); + FileUtils.touch(probe); + t2 = Files.getLastModifiedTime(probe); + if (wait < 100_000_000L) { + wait = wait * 2; } } - return c.getFsTimestampResolution(); + return Optional + .of(Duration.between(t1.toInstant(), t2.toInstant())); + } catch (IOException | TimeoutException e) { + LOG.error(e.getLocalizedMessage(), e); + } catch (InterruptedException e) { + LOG.error(e.getLocalizedMessage(), e); + Thread.currentThread().interrupt(); + } finally { + deleteProbe(probe); + } + return Optional.empty(); + } - } catch (IOException | InterruptedException e) { - LOG.warn(e.getMessage(), e); - return FALLBACK_TIMESTAMP_RESOLUTION; + private static void checkTimeout(FileStore s, long start) + throws TimeoutException { + if (System.nanoTime() - start >= FALLBACK_TIMESTAMP_RESOLUTION + .toNanos()) { + throw new TimeoutException(MessageFormat.format(JGitText + .get().timeoutMeasureFsTimestampResolution, + s.toString())); + } + } + private static void deleteProbe(Path probe) { + if (Files.exists(probe)) { + try { + Files.delete(probe); + } catch (IOException e) { + LOG.error(e.getLocalizedMessage(), e); + } } } - private Duration fsTimestampResolution; + private final @NonNull Duration fsTimestampResolution; + @NonNull Duration getFsTimestampResolution() { return fsTimestampResolution; } - private FileStoreAttributeCache(FileStore s, Path dir) - throws IOException, InterruptedException { - Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ - Files.createFile(probe); - try { - long start = System.nanoTime(); - FileTime startTime = Files.getLastModifiedTime(probe); - FileTime actTime = startTime; - long sleepTime = 512; - while (actTime.compareTo(startTime) <= 0) { - TimeUnit.NANOSECONDS.sleep(sleepTime); - if (timeout(start)) { - LOG.warn(MessageFormat.format(JGitText - .get().timeoutMeasureFsTimestampResolution, - s.toString())); - fsTimestampResolution = FALLBACK_TIMESTAMP_RESOLUTION; - return; - } - FileUtils.touch(probe); - actTime = Files.getLastModifiedTime(probe); - // limit sleep time to max. 100ms - if (sleepTime < 100_000_000L) { - sleepTime = sleepTime * 2; - } - } - fsTimestampResolution = Duration.between(startTime.toInstant(), - actTime.toInstant()); - } catch (AccessDeniedException e) { - LOG.error(e.getLocalizedMessage(), e); - } finally { - Files.delete(probe); - } - } - - private static boolean timeout(long start) { - return System.nanoTime() - start >= FALLBACK_TIMESTAMP_RESOLUTION - .toNanos(); + private FileStoreAttributeCache( + @NonNull Duration fsTimestampResolution) { + this.fsTimestampResolution = fsTimestampResolution; } @SuppressWarnings("nls") @@ -292,6 +365,20 @@ public static FS detect() { return detect(null); } + /** + * Whether FileStore attribute cache entries should be determined + * asynchronously + * + * @param asynch + * whether FileStore attribute cache entries should be determined + * asynchronously. If false access to cached attributes may block + * for some seconds for the first call per FileStore + * @since 5.1.9 + */ + public static void setAsyncfileStoreAttrCache(boolean asynch) { + FileStoreAttributeCache.setBackground(asynch); + } + /** * Auto-detect the appropriate file system abstraction, taking into account * the presence of a Cygwin installation on the system. Using jgit in From 121c957405d96b75e0e645717913d6c2786306c4 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 21 Jun 2019 17:58:56 +0200 Subject: [PATCH 13/47] Add support for nanoseconds and microseconds for Config#getTimeUnit Change-Id: I0a5828438810dd23790cba52d7ae2e055c6a3fc9 Signed-off-by: Matthias Sohn --- .../tst/org/eclipse/jgit/lib/ConfigTest.java | 12 ++++++++++++ .../eclipse/jgit/lib/DefaultTypedConfigGetter.java | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java index c4c4da816..2d0fe86f9 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java @@ -52,6 +52,8 @@ import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.eclipse.jgit.util.FileUtils.pathToString; import static org.junit.Assert.assertArrayEquals; @@ -928,8 +930,18 @@ private static Config parse(String content, Config baseConfig) @Test public void testTimeUnit() throws ConfigInvalidException { + assertEquals(0, parseTime("0", NANOSECONDS)); + assertEquals(2, parseTime("2ns", NANOSECONDS)); + assertEquals(200, parseTime("200 nanoseconds", NANOSECONDS)); + + assertEquals(0, parseTime("0", MICROSECONDS)); + assertEquals(2, parseTime("2us", MICROSECONDS)); + assertEquals(2, parseTime("2000 nanoseconds", MICROSECONDS)); + assertEquals(200, parseTime("200 microseconds", MICROSECONDS)); + assertEquals(0, parseTime("0", MILLISECONDS)); assertEquals(2, parseTime("2ms", MILLISECONDS)); + assertEquals(2, parseTime("2000microseconds", MILLISECONDS)); assertEquals(200, parseTime("200 milliseconds", MILLISECONDS)); assertEquals(0, parseTime("0s", SECONDS)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java index 891c7f23b..001ae93a0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java @@ -226,6 +226,14 @@ public long getTimeUnit(Config config, String section, String subsection, inputUnit = wantUnit; inputMul = 1; + } else if (match(unitName, "ns", "nanoseconds")) { //$NON-NLS-1$ //$NON-NLS-2$ + inputUnit = TimeUnit.NANOSECONDS; + inputMul = 1; + + } else if (match(unitName, "us", "microseconds")) { //$NON-NLS-1$ //$NON-NLS-2$ + inputUnit = TimeUnit.MICROSECONDS; + inputMul = 1; + } else if (match(unitName, "ms", "milliseconds")) { //$NON-NLS-1$ //$NON-NLS-2$ inputUnit = TimeUnit.MILLISECONDS; inputMul = 1; From e54fde8616f53a6fab5201db4a0c7e67dd0145c5 Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Tue, 16 Jul 2019 11:19:09 +0900 Subject: [PATCH 14/47] Bazel: Remove FileTreeIteratorWithTimeControl from BUILD file FileTreeIteratorWithTimeControl was deleted in a024759, but was not removed from the BUILD file, thus causing the bazel build to fail. Change-Id: I892c0ffcac947298d0d6009374ee2c5d9afefb66 Signed-off-by: David Pursehouse --- org.eclipse.jgit.test/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/org.eclipse.jgit.test/BUILD b/org.eclipse.jgit.test/BUILD index bbda838f0..ac8c1914f 100644 --- a/org.eclipse.jgit.test/BUILD +++ b/org.eclipse.jgit.test/BUILD @@ -19,7 +19,6 @@ HELPERS = glob(["src/**/*.java"]) + [PKG + c for c in [ "revwalk/RevQueueTestCase.java", "revwalk/RevWalkTestCase.java", "transport/SpiTransport.java", - "treewalk/FileTreeIteratorWithTimeControl.java", "treewalk/filter/AlwaysCloneTreeFilter.java", "test/resources/SampleDataRepositoryTestCase.java", "util/CPUTimeStopWatch.java", From 0aa8eca5f1c7a6d09774727f088ee8ca6f3e1760 Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Tue, 16 Jul 2019 11:41:05 +0900 Subject: [PATCH 15/47] Bazel: Add missing dependencies for ArchiveCommandTest Dependencies on commons-compress, xz, and jgit-archive are required for the build to succeed. Change-Id: I42f3721078a240ad93b8dcab909e66b9bfff0b56 Signed-off-by: David Pursehouse --- lib/BUILD | 1 + org.eclipse.jgit.test/tests.bzl | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/lib/BUILD b/lib/BUILD index ebba0daf4..b188265a8 100644 --- a/lib/BUILD +++ b/lib/BUILD @@ -12,6 +12,7 @@ java_library( visibility = [ "//org.eclipse.jgit.archive:__pkg__", "//org.eclipse.jgit.pgm.test:__pkg__", + "//org.eclipse.jgit.test:__pkg__", ], exports = ["@commons-compress//jar"], ) diff --git a/org.eclipse.jgit.test/tests.bzl b/org.eclipse.jgit.test/tests.bzl index bc06e3ef4..4963ecf9f 100644 --- a/org.eclipse.jgit.test/tests.bzl +++ b/org.eclipse.jgit.test/tests.bzl @@ -41,6 +41,12 @@ def tests(tests): additional_deps = [ "//lib:jsch", ] + if src.endswith("ArchiveCommandTest.java"): + additional_deps = [ + "//lib:commons-compress", + "//lib:xz", + "//org.eclipse.jgit.archive:jgit-archive", + ] heap_size = "-Xmx256m" if src.endswith("HugeCommitMessageTest.java"): From 74da1b2701fc3f736e53b25936dd566889097f8e Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Tue, 18 Jun 2019 09:17:16 +0200 Subject: [PATCH 16/47] Update bazlets to latest version This is needed to make build tool chain compatible with the latest Bazel releases. Change-Id: I9822b5fe5f934457e6069217d687b3cf4764b7b7 Signed-off-by: David Ostrovsky --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 9d723d28e..649f2b2e9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,7 +2,7 @@ workspace(name = "jgit") load("//tools:bazlets.bzl", "load_bazlets") -load_bazlets(commit = "3afbeab55ece585dbfc7a980bf7214b24ddbbe86") +load_bazlets(commit = "8528a0df69dadf6311d8d3f81c1b693afda8bcf1") load( "@com_googlesource_gerrit_bazlets//tools:maven_jar.bzl", From bbef67e8d02e9baff1d23f98e7252ebba1221b84 Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Tue, 18 Jun 2019 09:37:51 +0200 Subject: [PATCH 17/47] Bazel: Fix lint warning flagged by buildifier This change is fixing confusing name warning: [1]. ./org.eclipse.jgit.test/tests.bzl:12: confusing-name: Never use 'l', 'I', or 'O' as names (they're too easily confused with 'I', 'l', or '0'). And is also fixing: "All calls to rules or macros should pass arguments by keyword position argument" warning: [2]. ./org.eclipse.jgit.test/BUILD:42: positional-args: All calls to rules or macros should pass arguments by keyword (arg_name=value) syntax. [1] https://github.com/bazelbuild/buildtools/blob/master/WARNINGS.md#confusing-name [2] https://github.com/bazelbuild/buildtools/blob/master/WARNINGS.md#positional-args Change-Id: If5c28ec8a1ddc1d1b1035bd07b838a2a564aea4f Signed-off-by: David Ostrovsky --- org.eclipse.jgit.test/BUILD | 2 +- org.eclipse.jgit.test/tests.bzl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/org.eclipse.jgit.test/BUILD b/org.eclipse.jgit.test/BUILD index ac8c1914f..b06778eed 100644 --- a/org.eclipse.jgit.test/BUILD +++ b/org.eclipse.jgit.test/BUILD @@ -30,7 +30,7 @@ DATA = [ PKG + "lib/sorttest.gitindex.dat", ] -tests(glob( +tests(tests = glob( ["tst/**/*.java"], exclude = HELPERS + DATA, )) diff --git a/org.eclipse.jgit.test/tests.bzl b/org.eclipse.jgit.test/tests.bzl index 4963ecf9f..7ff90d484 100644 --- a/org.eclipse.jgit.test/tests.bzl +++ b/org.eclipse.jgit.test/tests.bzl @@ -8,14 +8,14 @@ def tests(tests): name = src[len("tst/"):len(src) - len(".java")].replace("/", "_") labels = [] if name.startswith("org_eclipse_jgit_"): - l = name[len("org.eclipse.jgit_"):] - if l.startswith("internal_storage_"): - l = l[len("internal.storage_"):] - i = l.find("_") - if i > 0: - labels.append(l[:i]) + package = name[len("org.eclipse.jgit_"):] + if package.startswith("internal_storage_"): + package = package[len("internal.storage_"):] + index = package.find("_") + if index > 0: + labels.append(package[:index]) else: - labels.append(i) + labels.append(index) if "lib" not in labels: labels.append("lib") From 0966731cad37fab239d458735c14be012cc70958 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 10 Jun 2019 23:35:20 +0200 Subject: [PATCH 18/47] Increase bazel timeout for long running tests EolRepositoryTest and GcCommitSelectionTest timed out frequently when running unit tests using bazel with the default timeout "moderate" (300s). Increase timeout of these tests to "long" (900s). Change-Id: I43588cf950f55b50f868d9fe9c66d22bd428a54c Signed-off-by: Matthias Sohn --- org.eclipse.jgit.test/tests.bzl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/org.eclipse.jgit.test/tests.bzl b/org.eclipse.jgit.test/tests.bzl index 7ff90d484..345da8110 100644 --- a/org.eclipse.jgit.test/tests.bzl +++ b/org.eclipse.jgit.test/tests.bzl @@ -7,6 +7,7 @@ def tests(tests): for src in tests: name = src[len("tst/"):len(src) - len(".java")].replace("/", "_") labels = [] + timeout = "moderate" if name.startswith("org_eclipse_jgit_"): package = name[len("org.eclipse.jgit_"):] if package.startswith("internal_storage_"): @@ -51,6 +52,8 @@ def tests(tests): heap_size = "-Xmx256m" if src.endswith("HugeCommitMessageTest.java"): heap_size = "-Xmx512m" + if src.endswith("EolRepositoryTest.java") or src.endswith("GcCommitSelectionTest.java"): + timeout = "long" junit_tests( name = name, @@ -68,4 +71,5 @@ def tests(tests): ], flaky = flaky, jvm_flags = [heap_size, "-Dfile.encoding=UTF-8"], + timeout = timeout, ) From 16760c3e9a242f21538c2f4da1aad96953cafc37 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 21 Jun 2019 18:12:14 +0200 Subject: [PATCH 19/47] Persist filesystem timestamp resolution and allow manual configuration To enable persisting filesystem timestamp resolution per FileStore add a new config section to the user global git configuration: - Config section is "filesystem" - Config subsection is concatenation of - Java vendor (system property "java.vm.vendor") - runtime version (system property "java.vm.version") - FileStore's name - separated by '|' e.g. "AdoptOpenJDK|1.8.0_212-b03|/dev/disk1s1" The prefix is needed since some Java versions do not expose the full timestamp resolution of the underlying filesystem. This may also depend on the underlying operating system hence concrete key values may not be portable. - Config key for timestamp resolution is "timestampResolution" as a time value, supported time units are those supported by DefaultTypedConfigGetter#getTimeUnit If timestamp resolution is already configured for a given FileStore the configured value is used instead of measuring the resolution. When timestamp resolution was measured it is persisted in the user global git configuration. Example: [filesystem "AdoptOpenJDK|1.8.0_212-b03|/dev/disk1s1"] timestampResolution = 1 seconds If locking the git config file fails retry saving the resolution up to 5 times in order to workaround races with another thread. In order to avoid stack overflow use the fallback filesystem timestamp resolution when loading FileBasedConfig which creates itself a FileSnapshot to help checking if the config changed. Note: - on some OSes Java 8,9 truncate to milliseconds or seconds, see https://bugs.openjdk.java.net/browse/JDK-8177809, fixed in Java 10 - UnixFileAttributes up to Java 12 truncates timestamp resolution to microseconds when converting the internal representation to FileTime exposed in the API, see https://bugs.openjdk.java.net/browse/JDK-8181493 - WindowsFileAttributes also provides only microsecond resolution up to Java 12 Hence do not attempt to manually configure a higher timestamp resolution than supported by the Java version being used at runtime. Bug: 546891 Bug: 548188 Change-Id: Iff91b8f9e6e5e2295e1463f87c8e95edf4abbcf8 Signed-off-by: Matthias Sohn --- org.eclipse.jgit/.settings/.api_filters | 22 ++++ .../eclipse/jgit/internal/JGitText.properties | 2 + .../org/eclipse/jgit/internal/JGitText.java | 2 + .../internal/storage/file/FileSnapshot.java | 40 ++++++- .../org/eclipse/jgit/lib/ConfigConstants.java | 12 +++ .../src/org/eclipse/jgit/lib/Constants.java | 11 ++ .../jgit/storage/file/FileBasedConfig.java | 4 +- .../src/org/eclipse/jgit/util/FS.java | 101 ++++++++++++++++-- 8 files changed, 183 insertions(+), 11 deletions(-) diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index d4c40788d..d313e92a0 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -22,6 +22,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 0a2f7029e..9af6cce59 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -104,6 +104,7 @@ cannotReadObjectsPath=Cannot read {0}/{1}: {2} cannotReadTree=Cannot read tree {0} cannotRebaseWithoutCurrentHead=Can not rebase without a current HEAD cannotResolveLocalTrackingRefForUpdating=Cannot resolve local tracking ref {0} for updating. +cannotSaveConfig=Cannot save config file ''{0}'' cannotSquashFixupWithoutPreviousCommit=Cannot {0} without previous commit. cannotStoreObjects=cannot store objects cannotResolveUniquelyAbbrevObjectId=Could not resolve uniquely the abbreviated object ID @@ -557,6 +558,7 @@ pushIsNotSupportedForBundleTransport=Push is not supported for bundle transport pushNotPermitted=push not permitted pushOptionsNotSupported=Push options not supported; received {0} rawLogMessageDoesNotParseAsLogEntry=Raw log message does not parse as log entry +readConfigFailed=Reading config file ''{0}'' failed readerIsRequired=Reader is required readingObjectsFromLocalRepositoryFailed=reading objects from local repository failed: {0} readTimedOut=Read timed out after {0} ms diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index bcd6d5ce2..79133d203 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -165,6 +165,7 @@ public static JGitText get() { /***/ public String cannotReadTree; /***/ public String cannotRebaseWithoutCurrentHead; /***/ public String cannotResolveLocalTrackingRefForUpdating; + /***/ public String cannotSaveConfig; /***/ public String cannotSquashFixupWithoutPreviousCommit; /***/ public String cannotStoreObjects; /***/ public String cannotResolveUniquelyAbbrevObjectId; @@ -618,6 +619,7 @@ public static JGitText get() { /***/ public String pushNotPermitted; /***/ public String pushOptionsNotSupported; /***/ public String rawLogMessageDoesNotParseAsLogEntry; + /***/ public String readConfigFailed; /***/ public String readerIsRequired; /***/ public String readingObjectsFromLocalRepositoryFailed; /***/ public String readTimedOut; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 0019c5f09..2c874ff59 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.File; import java.io.IOException; import java.nio.file.attribute.BasicFileAttributes; @@ -122,6 +123,22 @@ public static FileSnapshot save(File path) { return new FileSnapshot(path); } + /** + * Record a snapshot for a specific file path without using config file to + * get filesystem timestamp resolution. + *

+ * This method should be invoked before the file is accessed. It is used by + * FileBasedConfig to avoid endless recursion. + * + * @param path + * the path to later remember. The path's current status + * information is saved. + * @return the snapshot. + */ + public static FileSnapshot saveNoConfig(File path) { + return new FileSnapshot(path); + } + private static Object getFileKey(BasicFileAttributes fileAttributes) { Object fileKey = fileAttributes.fileKey(); return fileKey == null ? MISSING_FILEKEY : fileKey; @@ -177,13 +194,30 @@ public static FileSnapshot save(long modified) { * This method should be invoked before the file is accessed. * * @param path - * the path to later remember. The path's current status + * the path to remember meta data for. The path's current status * information is saved. */ protected FileSnapshot(File path) { + this(path, true); + } + + /** + * Record a snapshot for a specific file path. + *

+ * This method should be invoked before the file is accessed. + * + * @param path + * the path to remember meta data for. The path's current status + * information is saved. + * @param useConfig + * if {@code true} read filesystem time resolution from + * configuration file otherwise use fallback resolution + */ + protected FileSnapshot(File path, boolean useConfig) { this.lastRead = System.currentTimeMillis(); - this.fsTimestampResolution = FS - .getFsTimerResolution(path.toPath().getParent()); + this.fsTimestampResolution = useConfig + ? FS.getFsTimerResolution(path.toPath().getParent()) + : FALLBACK_TIMESTAMP_RESOLUTION; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(path); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index d4a0280da..4f636d455 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -432,4 +432,16 @@ public final class ConfigConstants { * @since 4.11 */ public static final String CONFIG_SECTION_LFS = "lfs"; + + /** + * The "filesystem" section + * @since 5.1.9 + */ + public static final String CONFIG_FILESYSTEM_SECTION = "filesystem"; + + /** + * The "timestampResolution" key + * @since 5.1.9 + */ + public static final String CONFIG_KEY_TIMESTAMP_RESOLUTION = "timestampResolution"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 4c5519696..94fc10038 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -52,6 +52,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; +import java.time.Duration; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.internal.JGitText; @@ -722,6 +723,16 @@ public static byte[] encode(String str) { */ public static final String LOCK_SUFFIX = ".lock"; //$NON-NLS-1$ + /** + * Fallback filesystem timestamp resolution used when we can't measure the + * resolution. The last modified time granularity of FAT filesystems is 2 + * seconds. + * + * @since 5.1.9 + */ + public static final Duration FALLBACK_TIMESTAMP_RESOLUTION = Duration + .ofMillis(2000); + private Constants() { // Hide the default constructor } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java index 93b3baa61..3a41643e6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java @@ -153,7 +153,9 @@ public void load() throws IOException, ConfigInvalidException { int retries = 0; while (true) { final FileSnapshot oldSnapshot = snapshot; - final FileSnapshot newSnapshot = FileSnapshot.save(getFile()); + // don't use config in this snapshot to avoid endless recursion + final FileSnapshot newSnapshot = FileSnapshot + .saveNoConfig(getFile()); try { final byte[] in = IO.readFully(getFile()); final ObjectId newHash = hash(in); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 2b4e5c78d..687c7a4fd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -87,9 +88,13 @@ import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.errors.CommandFailedException; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.treewalk.FileTreeIterator.FileEntry; import org.eclipse.jgit.treewalk.FileTreeIterator.FileModeStrategy; import org.eclipse.jgit.treewalk.WorkingTreeIterator.Entry; @@ -193,11 +198,9 @@ public int getRc() { } private static final class FileStoreAttributeCache { - /** - * The last modified time granularity of FAT filesystems is 2 seconds. - */ - private static final Duration FALLBACK_TIMESTAMP_RESOLUTION = Duration - .ofMillis(2000); + + private static final Duration UNDEFINED_RESOLUTION = Duration + .ofNanos(Long.MAX_VALUE); private static final Map attributeCache = new ConcurrentHashMap<>(); @@ -209,6 +212,10 @@ private static void setBackground(boolean async) { background.set(async); } + private static final String javaVersionPrefix = System + .getProperty("java.vm.vendor") + '|' //$NON-NLS-1$ + + System.getProperty("java.vm.version") + '|'; //$NON-NLS-1$ + private static Duration getFsTimestampResolution(Path file) { Path dir = Files.isDirectory(file) ? file : file.getParent(); FileStore s; @@ -280,6 +287,10 @@ private static Duration getFsTimestampResolution(Path file) { private static Optional measureFsTimestampResolution( FileStore s, Path dir) { + Duration configured = readFileTimeResolution(s); + if (!UNDEFINED_RESOLUTION.equals(configured)) { + return Optional.of(configured); + } Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ try { Files.createFile(probe); @@ -296,8 +307,9 @@ private static Optional measureFsTimestampResolution( wait = wait * 2; } } - return Optional - .of(Duration.between(t1.toInstant(), t2.toInstant())); + Duration resolution = Duration.between(t1.toInstant(), t2.toInstant()); + saveFileTimeResolution(s, resolution); + return Optional.of(resolution); } catch (IOException | TimeoutException e) { LOG.error(e.getLocalizedMessage(), e); } catch (InterruptedException e) { @@ -328,6 +340,81 @@ private static void deleteProbe(Path probe) { } } + private static Duration readFileTimeResolution(FileStore s) { + FileBasedConfig userConfig = SystemReader.getInstance() + .openUserConfig(null, FS.DETECTED); + try { + userConfig.load(); + } catch (IOException e) { + LOG.error(MessageFormat.format(JGitText.get().readConfigFailed, + userConfig.getFile().getAbsolutePath()), e); + } catch (ConfigInvalidException e) { + LOG.error(MessageFormat.format( + JGitText.get().repositoryConfigFileInvalid, + userConfig.getFile().getAbsolutePath(), + e.getMessage())); + } + Duration configured = Duration + .ofNanos(userConfig.getTimeUnit( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, + javaVersionPrefix + s.name(), + ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, + UNDEFINED_RESOLUTION.toNanos(), + TimeUnit.NANOSECONDS)); + return configured; + } + + private static void saveFileTimeResolution(FileStore s, + Duration resolution) { + FileBasedConfig userConfig = SystemReader.getInstance() + .openUserConfig(null, FS.DETECTED); + long nanos = resolution.toNanos(); + TimeUnit unit; + if (nanos < 200_000L) { + unit = TimeUnit.NANOSECONDS; + } else if (nanos < 200_000_000L) { + unit = TimeUnit.MICROSECONDS; + } else { + unit = TimeUnit.MILLISECONDS; + } + + final int max_retries = 5; + int retries = 0; + boolean succeeded = false; + long value = unit.convert(nanos, TimeUnit.NANOSECONDS); + while (!succeeded && retries < max_retries) { + try { + userConfig.load(); + userConfig.setString( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, + javaVersionPrefix + s.name(), + ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, + String.format("%d %s", //$NON-NLS-1$ + Long.valueOf(value), + unit.name().toLowerCase())); + userConfig.save(); + succeeded = true; + } catch (LockFailedException e) { + // race with another thread, wait a bit and try again + try { + retries++; + Thread.sleep(20); + } catch (InterruptedException e1) { + Thread.interrupted(); + } + } catch (IOException e) { + LOG.error(MessageFormat.format( + JGitText.get().cannotSaveConfig, + userConfig.getFile().getAbsolutePath()), e); + } catch (ConfigInvalidException e) { + LOG.error(MessageFormat.format( + JGitText.get().repositoryConfigFileInvalid, + userConfig.getFile().getAbsolutePath(), + e.getMessage())); + } + } + } + private final @NonNull Duration fsTimestampResolution; @NonNull From 8a0ed8a00425f5b101296c7636ea032c9485d77c Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 26 Jun 2019 01:11:12 +0200 Subject: [PATCH 20/47] Use FileChannel.open to touch file and set mtime to now Use options - StandardOpenOption.CREATE to create touched file if not existing - StandardOpenOption.SYNC to enforce synch of data and meta data changes - StandardOpenOption.WRITE Also set mtime explicitly in FileUtils#touch to the current system time. This should fix that the previous implementation didn't work on - locally cached Windows network share (CSC-CACHE filesystem) mapped as a drive - nfsv4 mounts on Linux and that it didn't create unborn file like Linux command "touch". Apache common's and Guava's touch() use the same approach. Immediately after creating the probe file used to measure timestamp resolution touch it. This ensures we always use the local system clock when measuring filesystem timestamp resolution. This should prevent that clock skew could influence the measured timestamp resolution in case of a mounted network filesystem. Bug: 548598 Change-Id: Iaeaf5967963f582395a195aa637b8188bfadac60 Signed-off-by: Matthias Sohn --- org.eclipse.jgit/.settings/.api_filters | 2 +- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 2 ++ .../src/org/eclipse/jgit/util/FileUtils.java | 12 ++++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index d313e92a0..5f58e7e45 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -153,7 +153,7 @@ - + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 687c7a4fd..64dbf5823 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -294,6 +294,8 @@ private static Optional measureFsTimestampResolution( Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ try { Files.createFile(probe); + // ensure we always use the local system clock + FileUtils.touch(probe); long wait = 512; long start = System.nanoTime(); FileTime t1 = Files.getLastModifiedTime(probe); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index 9bba6ca8a..9650602fe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -49,7 +49,7 @@ import java.io.File; import java.io.IOException; -import java.io.OutputStream; +import java.nio.channels.FileChannel; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.CopyOption; import java.nio.file.Files; @@ -57,6 +57,7 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributeView; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; @@ -66,6 +67,7 @@ import java.text.MessageFormat; import java.text.Normalizer; import java.text.Normalizer.Form; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -916,11 +918,13 @@ public static String pathToString(File file) { * @param f * the file to touch * @throws IOException - * @since 5.2.3 + * @since 5.1.8 */ public static void touch(Path f) throws IOException { - try (OutputStream fos = Files.newOutputStream(f)) { - // touch the file + try (FileChannel fc = FileChannel.open(f, StandardOpenOption.CREATE, + StandardOpenOption.APPEND, StandardOpenOption.SYNC)) { + // touch } + Files.setLastModifiedTime(f, FileTime.from(Instant.now())); } } From 09fec1a1024ab70322ab14ebd2f7aa4313c7c2e3 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 10 Jul 2019 10:19:01 +0200 Subject: [PATCH 21/47] Add debug trace for FileSnapshot Checking lastModified is time critical hence debug trace is the only way to analyze issues since debugging is impractical. Also add configuration for buffering of log4j output to reduce runtime impact when debug trace is on. Limit buffer to 1MiB and comment this configuration out since we may not always want to use buffering. Change-Id: Ib1a0537b67c8dc3fac994a77b42badd974ce6c97 Signed-off-by: Matthias Sohn --- .../tst-rsrc/log4j.properties | 2 + .../internal/storage/file/FileSnapshot.java | 61 +++++++++++++++---- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/org.eclipse.jgit.test/tst-rsrc/log4j.properties b/org.eclipse.jgit.test/tst-rsrc/log4j.properties index 14620ffae..a48a4022f 100644 --- a/org.eclipse.jgit.test/tst-rsrc/log4j.properties +++ b/org.eclipse.jgit.test/tst-rsrc/log4j.properties @@ -7,3 +7,5 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +#log4j.appender.fileLogger.bufferedIO = true +#log4j.appender.fileLogger.bufferSize = 1024 \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 2c874ff59..1b13ccf76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.internal.storage.file; import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; + import java.io.File; import java.io.IOException; import java.nio.file.attribute.BasicFileAttributes; @@ -57,6 +58,8 @@ import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.util.FS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Caches when a file was last read, making it possible to detect future edits. @@ -75,6 +78,8 @@ * file is less than 3 seconds ago. */ public class FileSnapshot { + private static final Logger LOG = LoggerFactory + .getLogger(FileSnapshot.class); /** * An unknown file size. * @@ -188,17 +193,19 @@ public static FileSnapshot save(long modified) { */ private final Object fileKey; + private final File file; + /** * Record a snapshot for a specific file path. *

* This method should be invoked before the file is accessed. * - * @param path + * @param file * the path to remember meta data for. The path's current status * information is saved. */ - protected FileSnapshot(File path) { - this(path, true); + protected FileSnapshot(File file) { + this(file, true); } /** @@ -206,30 +213,35 @@ protected FileSnapshot(File path) { *

* This method should be invoked before the file is accessed. * - * @param path + * @param file * the path to remember meta data for. The path's current status * information is saved. * @param useConfig * if {@code true} read filesystem time resolution from * configuration file otherwise use fallback resolution */ - protected FileSnapshot(File path, boolean useConfig) { + protected FileSnapshot(File file, boolean useConfig) { + this.file = file; this.lastRead = System.currentTimeMillis(); this.fsTimestampResolution = useConfig - ? FS.getFsTimerResolution(path.toPath().getParent()) + ? FS.getFsTimerResolution(file.toPath().getParent()) : FALLBACK_TIMESTAMP_RESOLUTION; BasicFileAttributes fileAttributes = null; try { - fileAttributes = FS.DETECTED.fileAttributes(path); + fileAttributes = FS.DETECTED.fileAttributes(file); } catch (IOException e) { - this.lastModified = path.lastModified(); - this.size = path.length(); + this.lastModified = file.lastModified(); + this.size = file.length(); this.fileKey = MISSING_FILEKEY; return; } this.lastModified = fileAttributes.lastModifiedTime().toMillis(); this.size = fileAttributes.size(); this.fileKey = getFileKey(fileAttributes); + LOG.debug("file={}, create new FileSnapshot: lastRead={} ms" //$NON-NLS-1$ + + ", lastModified={} ms, size={}, fileKey={}", //$NON-NLS-1$ + file, Long.valueOf(lastRead), Long.valueOf(lastModified), + Long.valueOf(size), fileKey); } private boolean sizeChanged; @@ -242,6 +254,7 @@ protected FileSnapshot(File path, boolean useConfig) { private FileSnapshot(long read, long modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { + this.file = null; this.lastRead = read; this.lastModified = modified; this.fsTimestampResolution = fsTimestampResolution; @@ -432,7 +445,14 @@ public String toString() { private boolean isRacyClean(long read) { // add a 10% safety margin long racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; - return wasRacyClean = (read - lastModified) * 1_000_000 <= racyNanos; + long delta = (read - lastModified) * 1_000_000; + wasRacyClean = delta <= racyNanos; + LOG.debug("file={}, isRacyClean={}, read={} ms, lastModified={} ms," //$NON-NLS-1$ + + " delta={} ns, racy<={} ns", //$NON-NLS-1$ + file, Boolean.valueOf(wasRacyClean), Long.valueOf(read), + Long.valueOf(lastModified), Long.valueOf(delta), + Long.valueOf(racyNanos)); + return wasRacyClean; } private boolean isModified(long currLastModified) { @@ -440,6 +460,9 @@ private boolean isModified(long currLastModified) { lastModifiedChanged = lastModified != currLastModified; if (lastModifiedChanged) { + LOG.debug("file={}, lastModified changed from {} to {}", //$NON-NLS-1$ + file, Long.valueOf(lastModified), + Long.valueOf(currLastModified)); return true; } @@ -447,26 +470,40 @@ private boolean isModified(long currLastModified) { // after the last modification that any new modifications // are certain to change the last modified time. if (cannotBeRacilyClean) { + LOG.debug("file={}, cannot be racily clean", file); //$NON-NLS-1$ return false; } if (!isRacyClean(lastRead)) { // Our last read should have marked cannotBeRacilyClean, // but this thread may not have seen the change. The read // of the volatile field lastRead should have fixed that. + LOG.debug("file={}, is unmodified", file); //$NON-NLS-1$ return false; } // We last read this path too close to its last observed // modification time. We may have missed a modification. // Scan again, to ensure we still see the same state. + LOG.debug("file={}, is racily clean", file); //$NON-NLS-1$ return true; } private boolean isFileKeyChanged(Object currFileKey) { - return currFileKey != MISSING_FILEKEY && !currFileKey.equals(fileKey); + boolean changed = currFileKey != MISSING_FILEKEY + && !currFileKey.equals(fileKey); + if (changed) { + LOG.debug("file={}, FileKey changed from {} to {}", //$NON-NLS-1$ + file, fileKey, currFileKey); + } + return changed; } private boolean isSizeChanged(long currSize) { - return currSize != UNKNOWN_SIZE && currSize != size; + boolean changed = (currSize != UNKNOWN_SIZE) && (currSize != size); + if (changed) { + LOG.debug("file={}, size changed from {} to {} bytes", //$NON-NLS-1$ + file, Long.valueOf(size), Long.valueOf(currSize)); + } + return changed; } } From be66222d1a607ef3feb3b076352e9b8251e0eec2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 26 Jun 2019 09:10:31 +0200 Subject: [PATCH 22/47] FS: ignore AccessDeniedException when measuring timestamp resolution It seems on cygwin creating a file under a writable directory can fail with AccessDeniedException. Log a warning in this case and fallback to worst case timestamp resolution of 2 seconds. Bug: 548648 Change-Id: Ic50c31ce9dc9ccadd4db5247df929418ac62d45c Signed-off-by: Matthias Sohn --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 64dbf5823..67634cbe7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -56,6 +56,7 @@ import java.io.OutputStream; import java.io.PrintStream; import java.nio.charset.Charset; +import java.nio.file.AccessDeniedException; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; @@ -312,6 +313,8 @@ private static Optional measureFsTimestampResolution( Duration resolution = Duration.between(t1.toInstant(), t2.toInstant()); saveFileTimeResolution(s, resolution); return Optional.of(resolution); + } catch (AccessDeniedException e) { + LOG.warn(e.getLocalizedMessage(), e); // see bug 548648 } catch (IOException | TimeoutException e) { LOG.error(e.getLocalizedMessage(), e); } catch (InterruptedException e) { From 79ede0c3227b0c5f5d7790b58a0881486e06d7cc Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 27 Jun 2019 00:39:24 +0200 Subject: [PATCH 23/47] Fix NPE in FS$FileStoreAttributeCache.getFsTimestampResolution Bug: 548682 Change-Id: I48840d3a68cf1db92c056d218a0d5ed0b9ea4c45 Signed-off-by: Matthias Sohn --- .../tst/org/eclipse/jgit/util/FSTest.java | 8 ++++++++ org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 1 + 2 files changed, 9 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index 59c8e31c0..6b3d58bae 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -65,6 +65,7 @@ import org.eclipse.jgit.errors.CommandFailedException; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.RepositoryCache; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -223,4 +224,11 @@ public void testFsTimestampResolution() throws Exception { } } } + + // bug 548682 + @Test + public void testRepoCacheRelativePathUnbornRepo() { + assertFalse(RepositoryCache.FileKey + .isGitRepository(new File("repo.git"), FS.DETECTED)); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 67634cbe7..a970a7df2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -218,6 +218,7 @@ private static void setBackground(boolean async) { + System.getProperty("java.vm.version") + '|'; //$NON-NLS-1$ private static Duration getFsTimestampResolution(Path file) { + file = file.toAbsolutePath(); Path dir = Files.isDirectory(file) ? file : file.getParent(); FileStore s; try { From 4db39f50742706091ee66207526cc801db40b780 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 3 Jul 2019 23:00:14 +0200 Subject: [PATCH 24/47] Workaround SecurityException in FS#getFsTimestampResolution On Android FS#getFsTimestampResolution always throws a SecurityException, handle this by falling back to the fallback timestamp resolution. Bug: 548947 Change-Id: I0ee6cb3c20e189bdc8d488434a930427ad6f2df2 Signed-off-by: Matthias Sohn --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index a970a7df2..6e3e336e9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -281,7 +281,7 @@ private static Duration getFsTimestampResolution(Path file) { } catch (IOException | InterruptedException | ExecutionException e) { LOG.error(e.getMessage(), e); - } catch (TimeoutException e) { + } catch (TimeoutException | SecurityException e) { // use fallback } return FALLBACK_TIMESTAMP_RESOLUTION; From 95e8264cc8d2689cec10b58fbf15149856000df4 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 3 Jul 2019 01:07:14 +0200 Subject: [PATCH 25/47] Use Instant instead of milliseconds for filesystem timestamp handling This enables higher file timestamp resolution on filesystems like ext4, Mac APFS (1ns) or NTFS (100ns) providing high timestamp resolution on filesystem level. Note: - on some OSes Java 8,9 truncate milliseconds, see https://bugs.openjdk.java.net/browse/JDK-8177809, fixed in Java 10 - UnixFileAttributes truncates timestamp resolution to microseconds when converting the internal representation to FileTime exposed in the API, see https://bugs.openjdk.java.net/browse/JDK-8181493 - WindowsFileAttributes also provides only microsecond resolution Change-Id: I25ffff31a3c6f725fc345d4ddc2f26da3b88f6f2 Signed-off-by: Matthias Sohn --- .../eclipse/jgit/http/server/FileSender.java | 8 +- .../jgit/http/server/ObjectFileServlet.java | 9 +- .../junit/LocalDiskRepositoryTestCase.java | 11 +- .../jgit/junit/RepositoryTestCase.java | 19 +-- .../org/eclipse/jgit/junit/time/TimeUtil.java | 99 ++++++++++++++ .../eclipse/jgit/pgm/debug/ShowDirCache.java | 22 +-- org.eclipse.jgit.test/META-INF/MANIFEST.MF | 5 +- .../org/eclipse/jgit/api/AddCommandTest.java | 2 +- .../eclipse/jgit/api/CheckoutCommandTest.java | 22 ++- .../eclipse/jgit/api/CommitCommandTest.java | 24 ++-- .../org/eclipse/jgit/api/DiffCommandTest.java | 4 +- .../eclipse/jgit/api/ResetCommandTest.java | 28 ++-- .../jgit/dircache/DirCacheBuilderTest.java | 15 +- .../jgit/dircache/DirCacheEntryTest.java | 7 +- .../storage/file/ConcurrentRepackTest.java | 14 +- .../storage/file/FileSnapshotTest.java | 23 ++-- .../internal/storage/file/GcTestCase.java | 7 +- .../storage/file/ObjectDirectoryTest.java | 20 +-- .../storage/file/PackFileSnapshotTest.java | 24 ++-- .../storage/file/RefDirectoryTest.java | 8 +- .../storage/file/T0003_BasicTest.java | 12 +- .../jgit/lib/IndexModificationTimesTest.java | 31 +++-- .../org/eclipse/jgit/lib/RacyGitTests.java | 14 +- .../org/eclipse/jgit/merge/MergerTest.java | 47 ++++--- .../jgit/transport/OpenSshConfigTest.java | 7 +- .../jgit/treewalk/FileTreeIteratorTest.java | 22 +-- .../tst/org/eclipse/jgit/util/FSTest.java | 8 +- org.eclipse.jgit/.settings/.api_filters | 62 +++++++++ .../eclipse/jgit/internal/JGitText.properties | 1 + .../src/org/eclipse/jgit/api/AddCommand.java | 5 +- .../org/eclipse/jgit/api/CommitCommand.java | 2 +- .../org/eclipse/jgit/api/ResetCommand.java | 2 +- .../eclipse/jgit/api/StashApplyCommand.java | 2 +- .../eclipse/jgit/api/StashCreateCommand.java | 3 +- .../org/eclipse/jgit/dircache/DirCache.java | 11 +- .../jgit/dircache/DirCacheCheckout.java | 13 +- .../eclipse/jgit/dircache/DirCacheEntry.java | 47 ++++++- .../org/eclipse/jgit/internal/JGitText.java | 1 + .../internal/storage/file/FileSnapshot.java | 128 ++++++++++++------ .../jgit/internal/storage/file/GC.java | 9 +- .../jgit/internal/storage/file/LockFile.java | 21 ++- .../jgit/internal/storage/file/PackFile.java | 7 +- .../org/eclipse/jgit/merge/ResolveMerger.java | 45 +++--- .../src/org/eclipse/jgit/transport/NetRC.java | 9 +- .../eclipse/jgit/transport/OpenSshConfig.java | 7 +- .../jgit/treewalk/FileTreeIterator.java | 9 +- .../jgit/treewalk/WorkingTreeIterator.java | 67 ++++++--- .../src/org/eclipse/jgit/util/FS.java | 71 +++++++++- .../src/org/eclipse/jgit/util/FS_Win32.java | 2 +- .../src/org/eclipse/jgit/util/FileUtils.java | 37 ++++- 50 files changed, 783 insertions(+), 290 deletions(-) create mode 100644 org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/time/TimeUtil.java diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/FileSender.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/FileSender.java index 05510a05b..946fb15a3 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/FileSender.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/FileSender.java @@ -58,12 +58,14 @@ import java.io.OutputStream; import java.io.RandomAccessFile; import java.text.MessageFormat; +import java.time.Instant; import java.util.Enumeration; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.util.FS; /** * Dumps a file over HTTP GET (or its information via HEAD). @@ -76,7 +78,7 @@ final class FileSender { private final RandomAccessFile source; - private final long lastModified; + private final Instant lastModified; private final long fileLen; @@ -89,7 +91,7 @@ final class FileSender { this.source = new RandomAccessFile(path, "r"); try { - this.lastModified = path.lastModified(); + this.lastModified = FS.DETECTED.lastModifiedInstant(path); this.fileLen = source.getChannel().size(); this.end = fileLen; } catch (IOException e) { @@ -114,7 +116,7 @@ void close() { } } - long getLastModified() { + Instant getLastModified() { return lastModified; } diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ObjectFileServlet.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ObjectFileServlet.java index 62f075c73..5a27be643 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ObjectFileServlet.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/ObjectFileServlet.java @@ -54,6 +54,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.time.Instant; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -76,7 +77,9 @@ static class Loose extends ObjectFileServlet { @Override String etag(FileSender sender) throws IOException { - return Long.toHexString(sender.getLastModified()); + Instant lastModified = sender.getLastModified(); + return Long.toHexString(lastModified.getEpochSecond()) + + Long.toHexString(lastModified.getNano()); } } @@ -145,7 +148,9 @@ private void serve(final HttpServletRequest req, try { final String etag = etag(sender); - final long lastModified = (sender.getLastModified() / 1000) * 1000; + // HTTP header Last-Modified header has a resolution of 1 sec, see + // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.29 + final long lastModified = sender.getLastModified().getEpochSecond(); String ifNoneMatch = req.getHeader(HDR_IF_NONE_MATCH); if (etag != null && etag.equals(ifNoneMatch)) { diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index 12b216950..50e6f2da5 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -51,6 +51,7 @@ import java.io.File; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -322,12 +323,13 @@ public static String indexState(Repository repo, int includedOptions) throws IllegalStateException, IOException { DirCache dc = repo.readDirCache(); StringBuilder sb = new StringBuilder(); - TreeSet timeStamps = new TreeSet<>(); + TreeSet timeStamps = new TreeSet<>(); // iterate once over the dircache just to collect all time stamps if (0 != (includedOptions & MOD_TIME)) { - for (int i=0; i + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.junit.time; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; + +import org.eclipse.jgit.util.FS; + +/** + * Utility methods for handling timestamps + */ +public class TimeUtil { + /** + * Set the lastModified time of a given file by adding a given offset to the + * current lastModified time + * + * @param path + * path of a file to set last modified + * @param offsetMillis + * offset in milliseconds, if negative the new lastModified time + * is offset before the original lastModified time, otherwise + * after the original time + * @return the new lastModified time + */ + public static Instant setLastModifiedWithOffset(Path path, + long offsetMillis) { + Instant mTime = FS.DETECTED.lastModifiedInstant(path) + .plusMillis(offsetMillis); + try { + Files.setLastModifiedTime(path, FileTime.from(mTime)); + return mTime; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Set the lastModified time of file a to the one from file b + * + * @param a + * file to set lastModified time + * @param b + * file to read lastModified time from + */ + public static void setLastModifiedOf(Path a, Path b) { + Instant mTime = FS.DETECTED.lastModifiedInstant(b); + try { + Files.setLastModifiedTime(a, FileTime.from(mTime)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowDirCache.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowDirCache.java index 7f99d76bd..14a60a3b4 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowDirCache.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowDirCache.java @@ -48,8 +48,10 @@ import static java.lang.Integer.valueOf; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Locale; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -67,25 +69,27 @@ class ShowDirCache extends TextBuiltin { /** {@inheritDoc} */ @Override protected void run() throws Exception { - final SimpleDateFormat fmt; - fmt = new SimpleDateFormat("yyyy-MM-dd,HH:mm:ss.SSS"); //$NON-NLS-1$ + final DateTimeFormatter fmt = DateTimeFormatter + .ofPattern("yyyy-MM-dd,HH:mm:ss.nnnnnnnnn") //$NON-NLS-1$ + .withLocale(Locale.getDefault()) + .withZone(ZoneId.systemDefault()); final DirCache cache = db.readDirCache(); for (int i = 0; i < cache.getEntryCount(); i++) { final DirCacheEntry ent = cache.getEntry(i); final FileMode mode = FileMode.fromBits(ent.getRawMode()); final int len = ent.getLength(); - long lastModified = ent.getLastModified(); - final Date mtime = new Date(lastModified); + Instant mtime = ent.getLastModifiedInstant(); final int stage = ent.getStage(); outw.print(mode); outw.format(" %6d", valueOf(len)); //$NON-NLS-1$ outw.print(' '); - if (millis) - outw.print(lastModified); - else + if (millis) { + outw.print(mtime.toEpochMilli()); + } else { outw.print(fmt.format(mtime)); + } outw.print(' '); outw.print(ent.getObjectId().name()); outw.print(' '); diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index f469173aa..89ce34dbf 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -40,6 +40,7 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", org.eclipse.jgit.internal.storage.reftable;version="[5.1.9,5.2.0)", org.eclipse.jgit.internal.storage.reftree;version="[5.1.9,5.2.0)", org.eclipse.jgit.junit;version="[5.1.9,5.2.0)", + org.eclipse.jgit.junit.time;version="[5.1.9,5.2.0)", org.eclipse.jgit.lfs;version="[5.1.9,5.2.0)", org.eclipse.jgit.lib;version="[5.1.9,5.2.0)", org.eclipse.jgit.merge;version="[5.1.9,5.2.0)", @@ -62,12 +63,12 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", org.eclipse.jgit.util;version="[5.1.9,5.2.0)", org.eclipse.jgit.util.io;version="[5.1.9,5.2.0)", org.eclipse.jgit.util.sha1;version="[5.1.9,5.2.0)", - org.tukaani.xz;version="[1.6.0,2.0)", org.junit;version="[4.12,5.0.0)", org.junit.experimental.theories;version="[4.12,5.0.0)", org.junit.rules;version="[4.12,5.0.0)", org.junit.runner;version="[4.12,5.0.0)", org.junit.runners;version="[4.12,5.0.0)", - org.slf4j;version="[1.7.0,2.0.0)" + org.slf4j;version="[1.7.0,2.0.0)", + org.tukaani.xz;version="[1.6.0,2.0)" Require-Bundle: org.hamcrest.core;bundle-version="[1.1.0,2.0.0)", org.hamcrest.library;bundle-version="[1.1.0,2.0.0)" diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index 911f65914..c67c86a93 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -1266,7 +1266,7 @@ private static DirCacheEntry addEntryToBuilder(String path, File file, DirCacheEntry entry = new DirCacheEntry(path, stage); entry.setObjectId(id); entry.setFileMode(FileMode.REGULAR_FILE); - entry.setLastModified(file.lastModified()); + entry.setLastModified(FS.DETECTED.lastModifiedInstant(file)); entry.setLength((int) file.length()); builder.add(entry); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java index 71df59e1f..08ad7b8bc 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java @@ -43,6 +43,7 @@ */ package org.eclipse.jgit.api; +import static java.time.Instant.EPOCH; import static org.eclipse.jgit.lib.Constants.MASTER; import static org.eclipse.jgit.lib.Constants.R_HEADS; import static org.hamcrest.CoreMatchers.is; @@ -60,6 +61,9 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import org.eclipse.jgit.api.CheckoutResult.Status; import org.eclipse.jgit.api.CreateBranchCommand.SetupUpstreamMode; @@ -74,6 +78,7 @@ import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.time.TimeUtil; import org.eclipse.jgit.lfs.BuiltinLFS; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -86,6 +91,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.SystemReader; import org.junit.Before; @@ -360,14 +366,14 @@ public void testUpdateSmudgedEntries() throws Exception { File file = new File(db.getWorkTree(), "Test.txt"); long size = file.length(); - long mTime = file.lastModified() - 5000L; - assertTrue(file.setLastModified(mTime)); + Instant mTime = TimeUtil.setLastModifiedWithOffset(file.toPath(), + -5000L); DirCache cache = DirCache.lock(db.getIndexFile(), db.getFS()); DirCacheEntry entry = cache.getEntry("Test.txt"); assertNotNull(entry); entry.setLength(0); - entry.setLastModified(0); + entry.setLastModified(EPOCH); cache.write(); assertTrue(cache.commit()); @@ -375,10 +381,12 @@ public void testUpdateSmudgedEntries() throws Exception { entry = cache.getEntry("Test.txt"); assertNotNull(entry); assertEquals(0, entry.getLength()); - assertEquals(0, entry.getLastModified()); + assertEquals(EPOCH, entry.getLastModifiedInstant()); - db.getIndexFile().setLastModified( - db.getIndexFile().lastModified() - 5000); + Files.setLastModifiedTime(db.getIndexFile().toPath(), + FileTime.from(FS.DETECTED + .lastModifiedInstant(db.getIndexFile()) + .minusMillis(5000L))); assertNotNull(git.checkout().setName("test").call()); @@ -386,7 +394,7 @@ public void testUpdateSmudgedEntries() throws Exception { entry = cache.getEntry("Test.txt"); assertNotNull(entry); assertEquals(size, entry.getLength()); - assertEquals(mTime, entry.getLastModified()); + assertEquals(mTime, entry.getLastModifiedInstant()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java index 3a13aa5a4..a6072a0f5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java @@ -61,6 +61,7 @@ import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.time.TimeUtil; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; @@ -311,11 +312,11 @@ public void commitSubmoduleUpdate() throws Exception { public void commitUpdatesSmudgedEntries() throws Exception { try (Git git = new Git(db)) { File file1 = writeTrashFile("file1.txt", "content1"); - assertTrue(file1.setLastModified(file1.lastModified() - 5000)); + TimeUtil.setLastModifiedWithOffset(file1.toPath(), -5000L); File file2 = writeTrashFile("file2.txt", "content2"); - assertTrue(file2.setLastModified(file2.lastModified() - 5000)); + TimeUtil.setLastModifiedWithOffset(file2.toPath(), -5000L); File file3 = writeTrashFile("file3.txt", "content3"); - assertTrue(file3.setLastModified(file3.lastModified() - 5000)); + TimeUtil.setLastModifiedWithOffset(file3.toPath(), -5000L); assertNotNull(git.add().addFilepattern("file1.txt") .addFilepattern("file2.txt").addFilepattern("file3.txt").call()); @@ -346,11 +347,12 @@ public void commitUpdatesSmudgedEntries() throws Exception { assertEquals(0, cache.getEntry("file2.txt").getLength()); assertEquals(0, cache.getEntry("file3.txt").getLength()); - long indexTime = db.getIndexFile().lastModified(); - db.getIndexFile().setLastModified(indexTime - 5000); + TimeUtil.setLastModifiedWithOffset(db.getIndexFile().toPath(), + -5000L); write(file1, "content4"); - assertTrue(file1.setLastModified(file1.lastModified() + 2500)); + + TimeUtil.setLastModifiedWithOffset(file1.toPath(), 2500L); assertNotNull(git.commit().setMessage("edit file").setOnly("file1.txt") .call()); @@ -368,9 +370,9 @@ public void commitUpdatesSmudgedEntries() throws Exception { public void commitIgnoresSmudgedEntryWithDifferentId() throws Exception { try (Git git = new Git(db)) { File file1 = writeTrashFile("file1.txt", "content1"); - assertTrue(file1.setLastModified(file1.lastModified() - 5000)); + TimeUtil.setLastModifiedWithOffset(file1.toPath(), -5000L); File file2 = writeTrashFile("file2.txt", "content2"); - assertTrue(file2.setLastModified(file2.lastModified() - 5000)); + TimeUtil.setLastModifiedWithOffset(file2.toPath(), -5000L); assertNotNull(git.add().addFilepattern("file1.txt") .addFilepattern("file2.txt").call()); @@ -399,11 +401,11 @@ public void commitIgnoresSmudgedEntryWithDifferentId() throws Exception { assertEquals(0, cache.getEntry("file1.txt").getLength()); assertEquals(0, cache.getEntry("file2.txt").getLength()); - long indexTime = db.getIndexFile().lastModified(); - db.getIndexFile().setLastModified(indexTime - 5000); + TimeUtil.setLastModifiedWithOffset(db.getIndexFile().toPath(), + -5000L); write(file1, "content5"); - assertTrue(file1.setLastModified(file1.lastModified() + 1000)); + TimeUtil.setLastModifiedWithOffset(file1.toPath(), 1000L); assertNotNull(git.commit().setMessage("edit file").setOnly("file1.txt") .call()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DiffCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DiffCommandTest.java index 43c3a8cf9..3a93839e8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DiffCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DiffCommandTest.java @@ -44,7 +44,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.File; @@ -55,6 +54,7 @@ import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.time.TimeUtil; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevWalk; @@ -230,7 +230,7 @@ public void testDiffWithNegativeLineCount() throws Exception { @Test public void testNoOutputStreamSet() throws Exception { File file = writeTrashFile("test.txt", "a"); - assertTrue(file.setLastModified(file.lastModified() - 5000)); + TimeUtil.setLastModifiedWithOffset(file.toPath(), -5000L); try (Git git = new Git(db)) { git.add().addFilepattern(".").call(); write(file, "b"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java index 2b97b307b..588387d3e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java @@ -42,6 +42,7 @@ */ package org.eclipse.jgit.api; +import static java.time.Instant.EPOCH; import static org.eclipse.jgit.api.ResetCommand.ResetType.HARD; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -52,6 +53,9 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import org.eclipse.jgit.api.ResetCommand.ResetType; import org.eclipse.jgit.api.errors.GitAPIException; @@ -246,13 +250,13 @@ public void testMixedReset() throws JGitInternalException, public void testMixedResetRetainsSizeAndModifiedTime() throws Exception { git = new Git(db); - writeTrashFile("a.txt", "a").setLastModified( - System.currentTimeMillis() - 60 * 1000); + Files.setLastModifiedTime(writeTrashFile("a.txt", "a").toPath(), + FileTime.from(Instant.now().minusSeconds(60))); assertNotNull(git.add().addFilepattern("a.txt").call()); assertNotNull(git.commit().setMessage("a commit").call()); - writeTrashFile("b.txt", "b").setLastModified( - System.currentTimeMillis() - 60 * 1000); + Files.setLastModifiedTime(writeTrashFile("b.txt", "b").toPath(), + FileTime.from(Instant.now().minusSeconds(60))); assertNotNull(git.add().addFilepattern("b.txt").call()); RevCommit commit2 = git.commit().setMessage("b commit").call(); assertNotNull(commit2); @@ -262,12 +266,12 @@ public void testMixedResetRetainsSizeAndModifiedTime() throws Exception { DirCacheEntry aEntry = cache.getEntry("a.txt"); assertNotNull(aEntry); assertTrue(aEntry.getLength() > 0); - assertTrue(aEntry.getLastModified() > 0); + assertTrue(aEntry.getLastModifiedInstant().compareTo(EPOCH) > 0); DirCacheEntry bEntry = cache.getEntry("b.txt"); assertNotNull(bEntry); assertTrue(bEntry.getLength() > 0); - assertTrue(bEntry.getLastModified() > 0); + assertTrue(bEntry.getLastModifiedInstant().compareTo(EPOCH) > 0); assertSameAsHead(git.reset().setMode(ResetType.MIXED) .setRef(commit2.getName()).call()); @@ -276,13 +280,17 @@ public void testMixedResetRetainsSizeAndModifiedTime() throws Exception { DirCacheEntry mixedAEntry = cache.getEntry("a.txt"); assertNotNull(mixedAEntry); - assertEquals(aEntry.getLastModified(), mixedAEntry.getLastModified()); - assertEquals(aEntry.getLastModified(), mixedAEntry.getLastModified()); + assertEquals(aEntry.getLastModifiedInstant(), + mixedAEntry.getLastModifiedInstant()); + assertEquals(aEntry.getLastModifiedInstant(), + mixedAEntry.getLastModifiedInstant()); DirCacheEntry mixedBEntry = cache.getEntry("b.txt"); assertNotNull(mixedBEntry); - assertEquals(bEntry.getLastModified(), mixedBEntry.getLastModified()); - assertEquals(bEntry.getLastModified(), mixedBEntry.getLastModified()); + assertEquals(bEntry.getLastModifiedInstant(), + mixedBEntry.getLastModifiedInstant()); + assertEquals(bEntry.getLastModifiedInstant(), + mixedBEntry.getLastModifiedInstant()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheBuilderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheBuilderTest.java index d12f302da..d9a420377 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheBuilderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheBuilderTest.java @@ -53,6 +53,7 @@ import static org.junit.Assert.fail; import java.io.File; +import java.time.Instant; import org.eclipse.jgit.events.IndexChangedEvent; import org.eclipse.jgit.events.IndexChangedListener; @@ -98,7 +99,7 @@ public void testBuildRejectsUnsetFileMode() throws Exception { public void testBuildOneFile_FinishWriteCommit() throws Exception { final String path = "a-file-path"; final FileMode mode = FileMode.REGULAR_FILE; - final long lastModified = 1218123387057L; + final Instant lastModified = Instant.ofEpochMilli(1218123387057L); final int length = 1342; final DirCacheEntry entOrig; { @@ -116,7 +117,7 @@ public void testBuildOneFile_FinishWriteCommit() throws Exception { assertEquals(ObjectId.zeroId(), entOrig.getObjectId()); assertEquals(mode.getBits(), entOrig.getRawMode()); assertEquals(0, entOrig.getStage()); - assertEquals(lastModified, entOrig.getLastModified()); + assertEquals(lastModified, entOrig.getLastModifiedInstant()); assertEquals(length, entOrig.getLength()); assertFalse(entOrig.isAssumeValid()); b.add(entOrig); @@ -138,7 +139,7 @@ public void testBuildOneFile_FinishWriteCommit() throws Exception { assertEquals(ObjectId.zeroId(), entOrig.getObjectId()); assertEquals(mode.getBits(), entOrig.getRawMode()); assertEquals(0, entOrig.getStage()); - assertEquals(lastModified, entOrig.getLastModified()); + assertEquals(lastModified, entOrig.getLastModifiedInstant()); assertEquals(length, entOrig.getLength()); assertFalse(entOrig.isAssumeValid()); } @@ -148,7 +149,7 @@ public void testBuildOneFile_FinishWriteCommit() throws Exception { public void testBuildOneFile_Commit() throws Exception { final String path = "a-file-path"; final FileMode mode = FileMode.REGULAR_FILE; - final long lastModified = 1218123387057L; + final Instant lastModified = Instant.ofEpochMilli(1218123387057L); final int length = 1342; final DirCacheEntry entOrig; { @@ -166,7 +167,7 @@ public void testBuildOneFile_Commit() throws Exception { assertEquals(ObjectId.zeroId(), entOrig.getObjectId()); assertEquals(mode.getBits(), entOrig.getRawMode()); assertEquals(0, entOrig.getStage()); - assertEquals(lastModified, entOrig.getLastModified()); + assertEquals(lastModified, entOrig.getLastModifiedInstant()); assertEquals(length, entOrig.getLength()); assertFalse(entOrig.isAssumeValid()); b.add(entOrig); @@ -186,7 +187,7 @@ public void testBuildOneFile_Commit() throws Exception { assertEquals(ObjectId.zeroId(), entOrig.getObjectId()); assertEquals(mode.getBits(), entOrig.getRawMode()); assertEquals(0, entOrig.getStage()); - assertEquals(lastModified, entOrig.getLastModified()); + assertEquals(lastModified, entOrig.getLastModifiedInstant()); assertEquals(length, entOrig.getLength()); assertFalse(entOrig.isAssumeValid()); } @@ -203,7 +204,7 @@ final class ReceivedEventMarkerException extends RuntimeException { final String path = "a-file-path"; final FileMode mode = FileMode.REGULAR_FILE; // "old" date in 2008 - final long lastModified = 1218123387057L; + final Instant lastModified = Instant.ofEpochMilli(1218123387057L); final int length = 1342; DirCacheEntry entOrig; boolean receivedEvent = false; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheEntryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheEntryTest.java index 86e285287..475819dbb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheEntryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/dircache/DirCacheEntryTest.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.dircache; +import static java.time.Instant.EPOCH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -188,7 +189,7 @@ private static void copyMetaDataHelper(boolean keepStage) { e.setAssumeValid(false); e.setCreationTime(2L); e.setFileMode(FileMode.EXECUTABLE_FILE); - e.setLastModified(3L); + e.setLastModified(EPOCH.plusMillis(3L)); e.setLength(100L); e.setObjectId(ObjectId .fromString("0123456789012345678901234567890123456789")); @@ -199,7 +200,7 @@ private static void copyMetaDataHelper(boolean keepStage) { f.setAssumeValid(true); f.setCreationTime(10L); f.setFileMode(FileMode.SYMLINK); - f.setLastModified(20L); + f.setLastModified(EPOCH.plusMillis(20L)); f.setLength(100000000L); f.setObjectId(ObjectId .fromString("1234567890123456789012345678901234567890")); @@ -212,7 +213,7 @@ private static void copyMetaDataHelper(boolean keepStage) { ObjectId.fromString("1234567890123456789012345678901234567890"), e.getObjectId()); assertEquals(FileMode.SYMLINK, e.getFileMode()); - assertEquals(20L, e.getLastModified()); + assertEquals(EPOCH.plusMillis(20L), e.getLastModifiedInstant()); assertEquals(100000000L, e.getLength()); if (keepStage) assertEquals(DirCacheEntry.STAGE_2, e.getStage()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java index 643daa5c9..6bec05673 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ConcurrentRepackTest.java @@ -56,6 +56,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.time.Instant; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; @@ -71,6 +72,7 @@ import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.WindowCacheConfig; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.junit.After; import org.junit.Before; @@ -235,7 +237,8 @@ private File[] pack(Repository src, RevObject... list) private static void write(File[] files, PackWriter pw) throws IOException { - final long begin = files[0].getParentFile().lastModified(); + final Instant begin = FS.DETECTED + .lastModifiedInstant(files[0].getParentFile()); NullProgressMonitor m = NullProgressMonitor.INSTANCE; try (OutputStream out = new BufferedOutputStream( @@ -252,7 +255,8 @@ private static void write(File[] files, PackWriter pw) } private static void delete(File[] list) throws IOException { - final long begin = list[0].getParentFile().lastModified(); + final Instant begin = FS.DETECTED + .lastModifiedInstant(list[0].getParentFile()); for (File f : list) { FileUtils.delete(f); assertFalse(f + " was removed", f.exists()); @@ -260,14 +264,14 @@ private static void delete(File[] list) throws IOException { touch(begin, list[0].getParentFile()); } - private static void touch(long begin, File dir) { - while (begin >= dir.lastModified()) { + private static void touch(Instant begin, File dir) throws IOException { + while (begin.compareTo(FS.DETECTED.lastModifiedInstant(dir)) >= 0) { try { Thread.sleep(25); } catch (InterruptedException ie) { // } - dir.setLastModified(System.currentTimeMillis()); + FS.DETECTED.setLastModified(dir.toPath(), Instant.now()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 6e458fbbf..467f6956b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -51,9 +51,11 @@ import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileTime; +import java.time.Instant; import java.util.ArrayList; import java.util.List; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.SystemReader; import org.junit.After; @@ -80,11 +82,12 @@ public void tearDown() throws Exception { FileUtils.delete(trash, FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); } - private static void waitNextSec(File f) { - long initialLastModified = f.lastModified(); + private static void waitNextTick(File f) throws IOException { + Instant initialLastModified = FS.DETECTED.lastModifiedInstant(f); do { - f.setLastModified(System.currentTimeMillis()); - } while (f.lastModified() == initialLastModified); + FS.DETECTED.setLastModified(f.toPath(), Instant.now()); + } while (FS.DETECTED.lastModifiedInstant(f) + .equals(initialLastModified)); } /** @@ -95,10 +98,10 @@ private static void waitNextSec(File f) { @Test public void testActuallyIsModifiedTrivial() throws Exception { File f1 = createFile("simple"); - waitNextSec(f1); + waitNextTick(f1); FileSnapshot save = FileSnapshot.save(f1); append(f1, (byte) 'x'); - waitNextSec(f1); + waitNextTick(f1); assertTrue(save.isModified(f1)); } @@ -113,7 +116,7 @@ public void testActuallyIsModifiedTrivial() throws Exception { @Test public void testNewFileWithWait() throws Exception { File f1 = createFile("newfile"); - waitNextSec(f1); + waitNextTick(f1); FileSnapshot save = FileSnapshot.save(f1); Thread.sleep(1500); assertTrue(save.isModified(f1)); @@ -146,8 +149,8 @@ public void testSimulatePackfileReplacement() throws Exception { File f2 = createFile("fool"); // Guarantees new inode x // wait on f2 since this method resets lastModified of the file // and leaves lastModified of f1 untouched - waitNextSec(f2); - waitNextSec(f2); + waitNextTick(f2); + waitNextTick(f2); FileTime timestamp = Files.getLastModifiedTime(f1.toPath()); FileSnapshot save = FileSnapshot.save(f1); Files.move(f2.toPath(), f1.toPath(), // Now "file" is inode x @@ -185,7 +188,7 @@ public void fileSnapshotEquals() throws Exception { // 0 sized FileSnapshot. FileSnapshot fs1 = FileSnapshot.MISSING_FILE; // UNKNOWN_SIZE FileSnapshot. - FileSnapshot fs2 = FileSnapshot.save(fs1.lastModified()); + FileSnapshot fs2 = FileSnapshot.save(fs1.lastModifiedInstant()); assertTrue(fs1.equals(fs2)); assertTrue(fs2.equals(fs1)); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java index d16998db5..eaa245b4e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTestCase.java @@ -147,9 +147,10 @@ protected RevCommit commitChain(int depth, int width) throws Exception { return tip; } - protected long lastModified(AnyObjectId objectId) throws IOException { - return repo.getFS().lastModified( - repo.getObjectDatabase().fileFor(objectId)); + protected long lastModified(AnyObjectId objectId) { + return repo.getFS() + .lastModifiedInstant(repo.getObjectDatabase().fileFor(objectId)) + .toEpochMilli(); } protected static void fsTick() throws InterruptedException, IOException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java index cbb73bb08..8cc06d93f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java @@ -65,6 +65,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; import org.junit.Assume; import org.junit.Rule; import org.junit.Test; @@ -157,13 +158,14 @@ public void testScanningForPackfiles() throws Exception { // To deal with racy-git situations JGit's Filesnapshot class will // report a file/folder potentially dirty if - // cachedLastReadTime-cachedLastModificationTime < 2500ms. This - // causes JGit to always rescan a file after modification. But: - // this was true only if the difference between current system time - // and cachedLastModification time was less than 2500ms. If the - // modification is more than 2500ms ago we may have reported a - // file/folder to be clean although it has not been rescanned. A - // Bug. To show the bug we sleep for more than 2500ms + // cachedLastReadTime-cachedLastModificationTime < filesystem + // timestamp resolution. This causes JGit to always rescan a file + // after modification. But: this was true only if the difference + // between current system time and cachedLastModification time was + // less than 2500ms. If the modification is more than 2500ms ago we + // may have reported a file/folder to be clean although it has not + // been rescanned. A bug. To show the bug we sleep for more than + // 2500ms Thread.sleep(2600); File[] ret = packsFolder.listFiles(new FilenameFilter() { @@ -173,7 +175,9 @@ public boolean accept(File dir, String name) { } }); assertTrue(ret != null && ret.length == 1); - Assume.assumeTrue(tmpFile.lastModified() == ret[0].lastModified()); + FS fs = db.getFS(); + Assume.assumeTrue(fs.lastModifiedInstant(tmpFile) + .equals(fs.lastModifiedInstant(ret[0]))); // all objects are in a new packfile but we will not detect it assertFalse(receivingDB.hasObject(unknownID)); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java index a1433e9fe..d5bc61a69 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java @@ -59,6 +59,7 @@ import java.nio.file.StandardOpenOption; //import java.nio.file.attribute.BasicFileAttributes; import java.text.ParseException; +import java.time.Instant; import java.util.Collection; import java.util.Iterator; import java.util.Random; @@ -80,6 +81,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.util.FS; import org.junit.Test; public class PackFileSnapshotTest extends RepositoryTestCase { @@ -188,7 +190,8 @@ public void testDetectModificationAlthoughSameSizeAndModificationtime() AnyObjectId chk1 = pf.getPackChecksum(); String name = pf.getPackName(); Long length = Long.valueOf(pf.getPackFile().length()); - long m1 = packFilePath.toFile().lastModified(); + FS fs = db.getFS(); + Instant m1 = fs.lastModifiedInstant(packFilePath); // Wait for a filesystem timer tick to enhance probability the rest of // this test is done before the filesystem timer ticks again. @@ -198,15 +201,15 @@ public void testDetectModificationAlthoughSameSizeAndModificationtime() // content and checksum are different since compression level differs AnyObjectId chk2 = repackAndCheck(6, name, length, chk1) .getPackChecksum(); - long m2 = packFilePath.toFile().lastModified(); - assumeFalse(m2 == m1); + Instant m2 = fs.lastModifiedInstant(packFilePath); + assumeFalse(m2.equals(m1)); // Repack to create packfile with same name, length. Lastmodified is // equal to the previous one because we are in the same filesystem timer // slot. Content and its checksum are different AnyObjectId chk3 = repackAndCheck(7, name, length, chk2) .getPackChecksum(); - long m3 = packFilePath.toFile().lastModified(); + Instant m3 = fs.lastModifiedInstant(packFilePath); // ask for an unknown git object to force jgit to rescan the list of // available packs. If we would ask for a known objectid then JGit would @@ -214,7 +217,7 @@ public void testDetectModificationAlthoughSameSizeAndModificationtime() db.getObjectDatabase().has(unknownID); assertEquals(chk3, getSinglePack(db.getObjectDatabase().getPacks()) .getPackChecksum()); - assumeTrue(m3 == m2); + assumeTrue(m3.equals(m2)); } // Try repacking so fast that we get two new packs which differ only in @@ -253,7 +256,8 @@ public void testDetectModificationAlthoughSameSizeAndModificationtimeAndFileKey( // Repack to create third packfile AnyObjectId chk3 = repackAndCheck(7, name, length, chk2) .getPackChecksum(); - long m3 = packFilePath.toFile().lastModified(); + FS fs = db.getFS(); + Instant m3 = fs.lastModifiedInstant(packFilePath); db.getObjectDatabase().has(unknownID); assertEquals(chk3, getSinglePack(db.getObjectDatabase().getPacks()) .getPackChecksum()); @@ -265,8 +269,8 @@ public void testDetectModificationAlthoughSameSizeAndModificationtimeAndFileKey( // Copy copy2 to packfile data to force modification of packfile without // changing the packfile's filekey. copyPack(packFileBasePath, ".copy2", ""); - long m2 = packFilePath.toFile().lastModified(); - assumeFalse(m3 == m2); + Instant m2 = fs.lastModifiedInstant(packFilePath); + assumeFalse(m3.equals(m2)); db.getObjectDatabase().has(unknownID); assertEquals(chk2, getSinglePack(db.getObjectDatabase().getPacks()) @@ -275,8 +279,8 @@ public void testDetectModificationAlthoughSameSizeAndModificationtimeAndFileKey( // Copy copy2 to packfile data to force modification of packfile without // changing the packfile's filekey. copyPack(packFileBasePath, ".copy1", ""); - long m1 = packFilePath.toFile().lastModified(); - assumeTrue(m2 == m1); + Instant m1 = fs.lastModifiedInstant(packFilePath); + assumeTrue(m2.equals(m1)); db.getObjectDatabase().has(unknownID); assertEquals(chk1, getSinglePack(db.getObjectDatabase().getPacks()) .getPackChecksum()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java index 5a2bd9c33..ac0a34ded 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java @@ -59,6 +59,7 @@ import java.io.File; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Map; @@ -78,6 +79,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevTag; +import org.eclipse.jgit.util.FS; import org.junit.Before; import org.junit.Test; @@ -1319,10 +1321,8 @@ private void writePackedRef(String name, AnyObjectId id) throws IOException { private void writePackedRefs(String content) throws IOException { File pr = new File(diskRepo.getDirectory(), "packed-refs"); write(pr, content); - - final long now = System.currentTimeMillis(); - final int oneHourAgo = 3600 * 1000; - pr.setLastModified(now - oneHourAgo); + FS fs = diskRepo.getFS(); + fs.setLastModified(pr.toPath(), Instant.now().minusSeconds(3600)); } private void deleteLooseRef(String name) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0003_BasicTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0003_BasicTest.java index 02073226d..a4509695d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0003_BasicTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0003_BasicTest.java @@ -59,6 +59,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.time.Instant; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -82,6 +83,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; import org.junit.Rule; @@ -790,12 +792,14 @@ private RevTag parseTag(AnyObjectId id) throws MissingObjectException, * * @param name * the file in the repository to force a time change on. + * @throws IOException */ - private void BUG_WorkAroundRacyGitIssues(String name) { + private void BUG_WorkAroundRacyGitIssues(String name) throws IOException { File path = new File(db.getDirectory(), name); - long old = path.lastModified(); + FS fs = db.getFS(); + Instant old = fs.lastModifiedInstant(path); long set = 1250379778668L; // Sat Aug 15 20:12:58 GMT-03:30 2009 - path.setLastModified(set); - assertTrue("time changed", old != path.lastModified()); + fs.setLastModified(path.toPath(), Instant.ofEpochMilli(set)); + assertFalse("time changed", old.equals(fs.lastModifiedInstant(path))); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexModificationTimesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexModificationTimesTest.java index b9bbbeb9e..a3634141c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexModificationTimesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexModificationTimesTest.java @@ -37,8 +37,12 @@ */ package org.eclipse.jgit.lib; +import static java.time.Instant.EPOCH; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.time.Instant; + import org.eclipse.jgit.api.Git; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -63,11 +67,11 @@ public void testLastModifiedTimes() throws Exception { DirCacheEntry entry = dc.getEntry(path); DirCacheEntry entry2 = dc.getEntry(path); - assertTrue("last modified shall not be zero!", - entry.getLastModified() != 0); + assertFalse("last modified shall not be the epoch!", + entry.getLastModifiedInstant().equals(EPOCH)); - assertTrue("last modified shall not be zero!", - entry2.getLastModified() != 0); + assertFalse("last modified shall not be the epoch!", + entry2.getLastModifiedInstant().equals(EPOCH)); writeTrashFile(path, "new content"); git.add().addFilepattern(path).call(); @@ -77,11 +81,11 @@ public void testLastModifiedTimes() throws Exception { entry = dc.getEntry(path); entry2 = dc.getEntry(path); - assertTrue("last modified shall not be zero!", - entry.getLastModified() != 0); + assertFalse("last modified shall not be the epoch!", + entry.getLastModifiedInstant().equals(EPOCH)); - assertTrue("last modified shall not be zero!", - entry2.getLastModified() != 0); + assertFalse("last modified shall not be the epoch!", + entry2.getLastModifiedInstant().equals(EPOCH)); } } @@ -97,7 +101,7 @@ public void testModify() throws Exception { DirCache dc = db.readDirCache(); DirCacheEntry entry = dc.getEntry(path); - long masterLastMod = entry.getLastModified(); + Instant masterLastMod = entry.getLastModifiedInstant(); git.checkout().setCreateBranch(true).setName("side").call(); @@ -110,7 +114,7 @@ public void testModify() throws Exception { dc = db.readDirCache(); entry = dc.getEntry(path); - long sideLastMode = entry.getLastModified(); + Instant sideLastMod = entry.getLastModifiedInstant(); Thread.sleep(2000); @@ -120,9 +124,10 @@ public void testModify() throws Exception { dc = db.readDirCache(); entry = dc.getEntry(path); - assertTrue("shall have equal mod time!", masterLastMod == sideLastMode); - assertTrue("shall not equal master timestamp!", - entry.getLastModified() == masterLastMod); + assertTrue("shall have equal mod time!", + masterLastMod.equals(sideLastMod)); + assertTrue("shall have equal master timestamp!", + entry.getLastModifiedInstant().equals(masterLastMod)); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java index d044e65cf..d3e4efef5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RacyGitTests.java @@ -50,12 +50,15 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.time.Instant; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.time.TimeUtil; import org.eclipse.jgit.treewalk.FileTreeIterator; import org.eclipse.jgit.treewalk.WorkingTreeOptions; +import org.eclipse.jgit.util.FS; import org.junit.Test; public class RacyGitTests extends RepositoryTestCase { @@ -74,8 +77,8 @@ public void testRacyGitDetection() throws Exception { // create two files File a = writeToWorkDir("a", "a"); File b = writeToWorkDir("b", "b"); - assertTrue(a.setLastModified(b.lastModified())); - assertTrue(b.setLastModified(b.lastModified())); + TimeUtil.setLastModifiedOf(a.toPath(), b.toPath()); + TimeUtil.setLastModifiedOf(b.toPath(), b.toPath()); // wait to ensure that file-modTimes and therefore index entry modTime // doesn't match the modtime of index-file after next persistance @@ -97,10 +100,11 @@ public void testRacyGitDetection() throws Exception { // filesystem timestamp resolution. By changing the index file // artificially, we create a fake racy situation. File updatedA = writeToWorkDir("a", "a2"); - long newLastModified = updatedA.lastModified() + 100; - assertTrue(updatedA.setLastModified(newLastModified)); + Instant newLastModified = TimeUtil + .setLastModifiedWithOffset(updatedA.toPath(), 100L); resetIndex(new FileTreeIterator(db)); - assertTrue(db.getIndexFile().setLastModified(newLastModified)); + FS.DETECTED.setLastModified(db.getIndexFile().toPath(), + newLastModified); DirCache dc = db.readDirCache(); // check index state: although racily clean a should not be reported as diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java index 7f5dba697..f22b7d6ad 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.merge; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.Instant.EPOCH; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -54,6 +55,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Map; @@ -1090,13 +1092,13 @@ public void checkLockedFilesToBeDeleted(MergeStrategy strategy) @Theory public void checkForCorrectIndex(MergeStrategy strategy) throws Exception { File f; - long lastTs4, lastTsIndex; + Instant lastTs4, lastTsIndex; Git git = Git.wrap(db); File indexFile = db.getIndexFile(); // Create initial content and remember when the last file was written. f = writeTrashFiles(false, "orig", "orig", "1\n2\n3", "orig", "orig"); - lastTs4 = FS.DETECTED.lastModified(f); + lastTs4 = FS.DETECTED.lastModifiedInstant(f); // add all files, commit and check this doesn't update any working tree // files and that the index is in a new file system timer tick. Make @@ -1109,8 +1111,9 @@ public void checkForCorrectIndex(MergeStrategy strategy) throws Exception { checkConsistentLastModified("0", "1", "2", "3", "4"); checkModificationTimeStampOrder("1", "2", "3", "4", "<.git/index"); assertEquals("Commit should not touch working tree file 4", lastTs4, - FS.DETECTED.lastModified(new File(db.getWorkTree(), "4"))); - lastTsIndex = FS.DETECTED.lastModified(indexFile); + FS.DETECTED + .lastModifiedInstant(new File(db.getWorkTree(), "4"))); + lastTsIndex = FS.DETECTED.lastModifiedInstant(indexFile); // Do modifications on the master branch. Then add and commit. This // should touch only "0", "2 and "3" @@ -1124,7 +1127,7 @@ public void checkForCorrectIndex(MergeStrategy strategy) throws Exception { checkConsistentLastModified("0", "1", "2", "3", "4"); checkModificationTimeStampOrder("1", "4", "*" + lastTs4, "<*" + lastTsIndex, "<0", "2", "3", "<.git/index"); - lastTsIndex = FS.DETECTED.lastModified(indexFile); + lastTsIndex = FS.DETECTED.lastModifiedInstant(indexFile); // Checkout a side branch. This should touch only "0", "2 and "3" fsTick(indexFile); @@ -1133,7 +1136,7 @@ public void checkForCorrectIndex(MergeStrategy strategy) throws Exception { checkConsistentLastModified("0", "1", "2", "3", "4"); checkModificationTimeStampOrder("1", "4", "*" + lastTs4, "<*" + lastTsIndex, "<0", "2", "3", ".git/index"); - lastTsIndex = FS.DETECTED.lastModified(indexFile); + lastTsIndex = FS.DETECTED.lastModifiedInstant(indexFile); // This checkout may have populated worktree and index so fast that we // may have smudged entries now. Check that we have the right content @@ -1146,13 +1149,13 @@ public void checkForCorrectIndex(MergeStrategy strategy) throws Exception { indexState(CONTENT)); fsTick(indexFile); f = writeTrashFiles(false, "orig", "orig", "1\n2\n3", "orig", "orig"); - lastTs4 = FS.DETECTED.lastModified(f); + lastTs4 = FS.DETECTED.lastModifiedInstant(f); fsTick(f); git.add().addFilepattern(".").call(); checkConsistentLastModified("0", "1", "2", "3", "4"); checkModificationTimeStampOrder("*" + lastTsIndex, "<0", "1", "2", "3", "4", "<.git/index"); - lastTsIndex = FS.DETECTED.lastModified(indexFile); + lastTsIndex = FS.DETECTED.lastModifiedInstant(indexFile); // Do modifications on the side branch. Touch only "1", "2 and "3" fsTick(indexFile); @@ -1163,7 +1166,7 @@ public void checkForCorrectIndex(MergeStrategy strategy) throws Exception { checkConsistentLastModified("0", "1", "2", "3", "4"); checkModificationTimeStampOrder("0", "4", "*" + lastTs4, "<*" + lastTsIndex, "<1", "2", "3", "<.git/index"); - lastTsIndex = FS.DETECTED.lastModified(indexFile); + lastTsIndex = FS.DETECTED.lastModifiedInstant(indexFile); // merge master and side. Should only touch "0," "2" and "3" fsTick(indexFile); @@ -1330,9 +1333,10 @@ private void checkConsistentLastModified(String... pathes) assertEquals( "IndexEntry with path " + path - + " has lastmodified with is different from the worktree file", - FS.DETECTED.lastModified(new File(workTree, path)), dc.getEntry(path) - .getLastModified()); + + " has lastmodified which is different from the worktree file", + FS.DETECTED.lastModifiedInstant(new File(workTree, path)), + dc.getEntry(path) + .getLastModifiedInstant()); } // Assert that modification timestamps of working tree files are as @@ -1341,21 +1345,22 @@ private void checkConsistentLastModified(String... pathes) // then this file must be younger then file i. A path "*" // represents a file with a modification time of // E.g. ("a", "b", " lastMod); - else + curMod.compareTo(lastMod) > 0); + } else { assertTrue("path " + p + " is older than predecesssor", - curMod >= lastMod); + curMod.compareTo(lastMod) >= 0); + } } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java index 19fcbfd7a..7777a3c99 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java @@ -57,10 +57,12 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; +import java.time.Instant; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.transport.OpenSshConfig.Host; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.SystemReader; import org.junit.Before; @@ -91,13 +93,14 @@ public void setUp() throws Exception { } private void config(String data) throws IOException { - long lastMtime = configFile.lastModified(); + FS fs = db.getFS(); + Instant lastMtime = fs.lastModifiedInstant(configFile); do { try (final OutputStreamWriter fw = new OutputStreamWriter( new FileOutputStream(configFile), UTF_8)) { fw.write(data); } - } while (lastMtime == configFile.lastModified()); + } while (lastMtime.equals(fs.lastModifiedInstant(configFile))); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java index 33e32cd81..9a0e7d2ea 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java @@ -52,6 +52,7 @@ import java.io.File; import java.io.IOException; import java.security.MessageDigest; +import java.time.Instant; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ResetCommand.ResetType; @@ -86,7 +87,7 @@ public class FileTreeIteratorTest extends RepositoryTestCase { private final String[] paths = { "a,", "a,b", "a/b", "a0b" }; - private long[] mtime; + private Instant[] mtime; @Override @Before @@ -99,11 +100,11 @@ public void setUp() throws Exception { // This should stress the sorting code better than doing it in // the correct order. // - mtime = new long[paths.length]; + mtime = new Instant[paths.length]; for (int i = paths.length - 1; i >= 0; i--) { final String s = paths[i]; writeTrashFile(s, s); - mtime[i] = FS.DETECTED.lastModified(new File(trash, s)); + mtime[i] = db.getFS().lastModifiedInstant(new File(trash, s)); } } @@ -199,7 +200,7 @@ public void testSimpleIterate() throws Exception { assertEquals(FileMode.REGULAR_FILE.getBits(), top.mode); assertEquals(paths[0], nameOf(top)); assertEquals(paths[0].length(), top.getEntryLength()); - assertEquals(mtime[0], top.getEntryLastModified()); + assertEquals(mtime[0], top.getEntryLastModifiedInstant()); top.next(1); assertFalse(top.first()); @@ -207,7 +208,7 @@ public void testSimpleIterate() throws Exception { assertEquals(FileMode.REGULAR_FILE.getBits(), top.mode); assertEquals(paths[1], nameOf(top)); assertEquals(paths[1].length(), top.getEntryLength()); - assertEquals(mtime[1], top.getEntryLastModified()); + assertEquals(mtime[1], top.getEntryLastModifiedInstant()); top.next(1); assertFalse(top.first()); @@ -222,7 +223,7 @@ public void testSimpleIterate() throws Exception { assertFalse(sub.eof()); assertEquals(paths[2], nameOf(sub)); assertEquals(paths[2].length(), subfti.getEntryLength()); - assertEquals(mtime[2], subfti.getEntryLastModified()); + assertEquals(mtime[2], subfti.getEntryLastModifiedInstant()); sub.next(1); assertTrue(sub.eof()); @@ -233,7 +234,7 @@ public void testSimpleIterate() throws Exception { assertEquals(FileMode.REGULAR_FILE.getBits(), top.mode); assertEquals(paths[3], nameOf(top)); assertEquals(paths[3].length(), top.getEntryLength()); - assertEquals(mtime[3], top.getEntryLastModified()); + assertEquals(mtime[3], top.getEntryLastModifiedInstant()); top.next(1); assertTrue(top.eof()); @@ -345,20 +346,21 @@ public void testIsModifiedSymlinkAsFile() throws Exception { @Test public void testIsModifiedFileSmudged() throws Exception { File f = writeTrashFile("file", "content"); + FS fs = db.getFS(); try (Git git = new Git(db)) { // The idea of this test is to check the smudged handling // Hopefully fsTick will make sure our entry gets smudged fsTick(f); writeTrashFile("file", "content"); - long lastModified = f.lastModified(); + Instant lastModified = fs.lastModifiedInstant(f); git.add().addFilepattern("file").call(); writeTrashFile("file", "conten2"); - f.setLastModified(lastModified); + fs.setLastModified(f.toPath(), lastModified); // We cannot trust this to go fast enough on // a system with less than one-second lastModified // resolution, so we force the index to have the // same timestamp as the file we look at. - db.getIndexFile().setLastModified(lastModified); + fs.setLastModified(db.getIndexFile().toPath(), lastModified); } DirCacheEntry dce = db.readDirCache().getEntry("file"); FileTreeIterator fti = new FileTreeIterator(trash, db.getFS(), db diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index 6b3d58bae..bde8a8a6b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.util; +import static java.time.Instant.EPOCH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -106,7 +107,7 @@ public void testSymlinkAttributes() throws IOException, InterruptedException { assertTrue(fs.exists(link)); String targetName = fs.readSymLink(link); assertEquals("å", targetName); - assertTrue(fs.lastModified(link) > 0); + assertTrue(fs.lastModifiedInstant(link).compareTo(EPOCH) > 0); assertTrue(fs.exists(link)); assertFalse(fs.canExecute(link)); assertEquals(2, fs.length(link)); @@ -119,8 +120,9 @@ public void testSymlinkAttributes() throws IOException, InterruptedException { // Now create the link target FileUtils.createNewFile(target); assertTrue(fs.exists(link)); - assertTrue(fs.lastModified(link) > 0); - assertTrue(fs.lastModified(target) > fs.lastModified(link)); + assertTrue(fs.lastModifiedInstant(link).compareTo(EPOCH) > 0); + assertTrue(fs.lastModifiedInstant(target) + .compareTo(fs.lastModifiedInstant(link)) > 0); assertFalse(fs.canExecute(link)); fs.setExecute(target, true); assertFalse(fs.canExecute(link)); diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 5f58e7e45..951a5e30f 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -8,6 +8,20 @@ + + + + + + + + + + + + + + @@ -130,6 +144,28 @@ + + + + + + + + + + + + + + + + + + + + + + @@ -137,12 +173,30 @@ + + + + + + + + + + + + + + + + + + @@ -150,6 +204,14 @@ + + + + + + + + diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 9af6cce59..6ac6955f0 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -561,6 +561,7 @@ rawLogMessageDoesNotParseAsLogEntry=Raw log message does not parse as log entry readConfigFailed=Reading config file ''{0}'' failed readerIsRequired=Reader is required readingObjectsFromLocalRepositoryFailed=reading objects from local repository failed: {0} +readLastModifiedFailed=Reading lastModified of {0} failed readTimedOut=Read timed out after {0} ms receivePackObjectTooLarge1=Object too large, rejecting the pack. Max object size limit is {0} bytes. receivePackObjectTooLarge2=Object too large ({0} bytes), rejecting the pack. Max object size limit is {1} bytes. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index f0408ab3d..a2cd4aeb2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import java.util.Collection; import java.util.LinkedList; @@ -228,7 +229,7 @@ public DirCache call() throws GitAPIException, NoFilepatternException { if (GITLINK != mode) { entry.setLength(f.getEntryLength()); - entry.setLastModified(f.getEntryLastModified()); + entry.setLastModified(f.getEntryLastModifiedInstant()); long len = f.getEntryContentLength(); // We read and filter the content multiple times. // f.getEntryContentLength() reads and filters the input and @@ -241,7 +242,7 @@ public DirCache call() throws GitAPIException, NoFilepatternException { } } else { entry.setLength(0); - entry.setLastModified(0); + entry.setLastModified(Instant.ofEpochSecond(0)); entry.setObjectId(f.getEntryObjectId()); } builder.add(entry); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index d07532c09..cea28fac1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -392,7 +392,7 @@ private DirCache createTemporaryIndex(ObjectId headId, DirCache index, final DirCacheEntry dcEntry = new DirCacheEntry(path); long entryLength = fTree.getEntryLength(); dcEntry.setLength(entryLength); - dcEntry.setLastModified(fTree.getEntryLastModified()); + dcEntry.setLastModified(fTree.getEntryLastModifiedInstant()); dcEntry.setFileMode(fTree.getIndexFileMode(dcTree)); boolean objectExists = (dcTree != null diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java index 13ce4e769..d7c9ad5e0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java @@ -422,7 +422,7 @@ private void resetIndex(ObjectId commitTree) throws IOException { DirCacheIterator.class); if (dcIter != null && dcIter.idEqual(cIter)) { DirCacheEntry indexEntry = dcIter.getDirCacheEntry(); - entry.setLastModified(indexEntry.getLastModified()); + entry.setLastModified(indexEntry.getLastModifiedInstant()); entry.setLength(indexEntry.getLength()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java index 01d070cbd..ff7c4c64b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java @@ -332,7 +332,7 @@ private void resetIndex(RevTree tree) throws IOException { DirCacheIterator.class); if (dcIter != null && dcIter.idEqual(cIter)) { DirCacheEntry indexEntry = dcIter.getDirCacheEntry(); - entry.setLastModified(indexEntry.getLastModified()); + entry.setLastModified(indexEntry.getLastModifiedInstant()); entry.setLength(indexEntry.getLength()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java index c32890d8a..0d010adb2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java @@ -300,7 +300,8 @@ public RevCommit call() throws GitAPIException { final DirCacheEntry entry = new DirCacheEntry( treeWalk.getRawPath()); entry.setLength(wtIter.getEntryLength()); - entry.setLastModified(wtIter.getEntryLastModified()); + entry.setLastModified( + wtIter.getEntryLastModifiedInstant()); entry.setFileMode(wtIter.getEntryFileMode()); long contentLength = wtIter.getEntryContentLength(); try (InputStream in = wtIter.openEntryStream()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java index 14653fe17..340214b06 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java @@ -497,8 +497,9 @@ else if (ver != 2) throw new CorruptObjectException(JGitText.get().DIRCHasTooManyEntries); snapshot = FileSnapshot.save(liveFile); - int smudge_s = (int) (snapshot.lastModified() / 1000); - int smudge_ns = ((int) (snapshot.lastModified() % 1000)) * 1000000; + // TODO (ms) combine smudge_s and smudge_ns into Duration + int smudge_s = (int) (snapshot.lastModifiedInstant().getEpochSecond()); + int smudge_ns = snapshot.lastModifiedInstant().getNano(); // Load the individual file entries. // @@ -679,8 +680,8 @@ void writeTo(File dir, OutputStream os) throws IOException { // so we use the current timestamp as a approximation. myLock.createCommitSnapshot(); snapshot = myLock.getCommitSnapshot(); - smudge_s = (int) (snapshot.lastModified() / 1000); - smudge_ns = ((int) (snapshot.lastModified() % 1000)) * 1000000; + smudge_s = (int) (snapshot.lastModifiedInstant().getEpochSecond()); + smudge_ns = snapshot.lastModifiedInstant().getNano(); } else { // Used in unit tests only smudge_ns = 0; @@ -1025,7 +1026,7 @@ private void updateSmudgedEntries() throws IOException { DirCacheEntry entry = iIter.getDirCacheEntry(); if (entry.isSmudged() && iIter.idEqual(fIter)) { entry.setLength(fIter.getEntryLength()); - entry.setLastModified(fIter.getEntryLastModified()); + entry.setLastModified(fIter.getEntryLastModifiedInstant()); } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index ca1e3ab27..5110d77dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -50,6 +50,7 @@ import java.io.OutputStream; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -425,8 +426,10 @@ void processEntry(CanonicalTreeParser m, DirCacheBuildIterator i, // update the timestamp of the index with the one from the // file if not set, as we are sure to be in sync here. DirCacheEntry entry = i.getDirCacheEntry(); - if (entry.getLastModified() == 0) - entry.setLastModified(f.getEntryLastModified()); + Instant mtime = entry.getLastModifiedInstant(); + if (mtime == null || mtime.equals(Instant.EPOCH)) { + entry.setLastModified(f.getEntryLastModifiedInstant()); + } keep(entry); } } else @@ -611,7 +614,7 @@ private void checkoutGitlink(String path, DirCacheEntry entry) File gitlinkDir = new File(repo.getWorkTree(), path); FileUtils.mkdirs(gitlinkDir, true); FS fs = repo.getFS(); - entry.setLastModified(fs.lastModified(gitlinkDir)); + entry.setLastModified(fs.lastModifiedInstant(gitlinkDir)); } private static ArrayList filterOut(ArrayList strings, @@ -1433,7 +1436,7 @@ public static void checkoutEntry(Repository repo, DirCacheEntry entry, } fs.createSymLink(f, target); entry.setLength(bytes.length); - entry.setLastModified(fs.lastModified(f)); + entry.setLastModified(fs.lastModifiedInstant(f)); return; } @@ -1502,7 +1505,7 @@ public static void checkoutEntry(Repository repo, DirCacheEntry entry, FileUtils.delete(tmpFile); } } - entry.setLastModified(fs.lastModified(f)); + entry.setLastModified(fs.lastModifiedInstant(f)); } // Run an external filter command diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java index 6b1d4f4d8..4b36f6b96 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java @@ -56,6 +56,7 @@ import java.nio.ByteBuffer; import java.security.MessageDigest; import java.text.MessageFormat; +import java.time.Instant; import java.util.Arrays; import org.eclipse.jgit.errors.CorruptObjectException; @@ -144,6 +145,7 @@ public class DirCacheEntry { /** Flags which are never stored to disk. */ private byte inCoreFlags; + // TODO (ms): use Instant to combine smudge_s and smudge_ns DirCacheEntry(final byte[] sharedInfo, final MutableInteger infoAt, final InputStream in, final MessageDigest md, final int smudge_s, final int smudge_ns) throws IOException { @@ -563,21 +565,50 @@ public void setCreationTime(long when) { * * @return last modification time of this file, in milliseconds since the * Java epoch (midnight Jan 1, 1970 UTC). + * @deprecated use {@link #getLastModifiedInstant()} instead */ + @Deprecated public long getLastModified() { return decodeTS(P_MTIME); } + /** + * Get the cached last modification date of this file. + *

+ * One of the indicators that the file has been modified by an application + * changing the working tree is if the last modification time for the file + * differs from the time stored in this entry. + * + * @return last modification time of this file. + * @since 5.1.9 + */ + public Instant getLastModifiedInstant() { + return decodeTSInstant(P_MTIME); + } + /** * Set the cached last modification date of this file, using milliseconds. * * @param when * new cached modification date of the file, in milliseconds. + * @deprecated use {@link #setLastModified(Instant)} instead */ + @Deprecated public void setLastModified(long when) { encodeTS(P_MTIME, when); } + /** + * Set the cached last modification date of this file. + * + * @param when + * new cached modification date of the file. + * @since 5.1.9 + */ + public void setLastModified(Instant when) { + encodeTS(P_MTIME, when); + } + /** * Get the cached size (mod 4 GB) (in bytes) of this file. *

@@ -692,7 +723,8 @@ public byte[] getRawPath() { @SuppressWarnings("nls") @Override public String toString() { - return getFileMode() + " " + getLength() + " " + getLastModified() + return getFileMode() + " " + getLength() + " " + + getLastModifiedInstant() + " " + getObjectId() + " " + getStage() + " " + getPathString() + "\n"; } @@ -750,12 +782,25 @@ private long decodeTS(int pIdx) { return 1000L * sec + ms; } + private Instant decodeTSInstant(int pIdx) { + final int base = infoOffset + pIdx; + final int sec = NB.decodeInt32(info, base); + final int nano = NB.decodeInt32(info, base + 4); + return Instant.ofEpochSecond(sec, nano); + } + private void encodeTS(int pIdx, long when) { final int base = infoOffset + pIdx; NB.encodeInt32(info, base, (int) (when / 1000)); NB.encodeInt32(info, base + 4, ((int) (when % 1000)) * 1000000); } + private void encodeTS(int pIdx, Instant when) { + final int base = infoOffset + pIdx; + NB.encodeInt32(info, base, (int) when.getEpochSecond()); + NB.encodeInt32(info, base + 4, when.getNano()); + } + private int getExtendedFlags() { if (isExtended()) return NB.decodeUInt16(info, infoOffset + P_FLAGS2) << 16; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 79133d203..38c063d07 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -622,6 +622,7 @@ public static JGitText get() { /***/ public String readConfigFailed; /***/ public String readerIsRequired; /***/ public String readingObjectsFromLocalRepositoryFailed; + /***/ public String readLastModifiedFailed; /***/ public String readTimedOut; /***/ public String receivePackObjectTooLarge1; /***/ public String receivePackObjectTooLarge2; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 1b13ccf76..fe3703719 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -48,10 +48,10 @@ import java.io.File; import java.io.IOException; import java.nio.file.attribute.BasicFileAttributes; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.time.Duration; -import java.util.Date; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -87,8 +87,14 @@ public class FileSnapshot { */ public static final long UNKNOWN_SIZE = -1; + private static final Instant UNKNOWN_TIME = Instant.ofEpochMilli(-1); + private static final Object MISSING_FILEKEY = new Object(); + private static final DateTimeFormatter dateFmt = DateTimeFormatter + .ofPattern("yyyy-MM-dd HH:mm:ss.nnnnnnnnn") //$NON-NLS-1$ + .withLocale(Locale.getDefault()).withZone(ZoneId.systemDefault()); + /** * A FileSnapshot that is considered to always be modified. *

@@ -96,8 +102,8 @@ public class FileSnapshot { * file, but only after {@link #isModified(File)} gets invoked. The returned * snapshot contains only invalid status information. */ - public static final FileSnapshot DIRTY = new FileSnapshot(-1, -1, - UNKNOWN_SIZE, Duration.ZERO, MISSING_FILEKEY); + public static final FileSnapshot DIRTY = new FileSnapshot(UNKNOWN_TIME, + UNKNOWN_TIME, UNKNOWN_SIZE, Duration.ZERO, MISSING_FILEKEY); /** * A FileSnapshot that is clean if the file does not exist. @@ -106,8 +112,8 @@ public class FileSnapshot { * file to be clean. {@link #isModified(File)} will return false if the file * path does not exist. */ - public static final FileSnapshot MISSING_FILE = new FileSnapshot(0, 0, 0, - Duration.ZERO, MISSING_FILEKEY) { + public static final FileSnapshot MISSING_FILE = new FileSnapshot( + Instant.EPOCH, Instant.EPOCH, 0, Duration.ZERO, MISSING_FILEKEY) { @Override public boolean isModified(File path) { return FS.DETECTED.exists(path); @@ -163,18 +169,41 @@ private static Object getFileKey(BasicFileAttributes fileAttributes) { * @param modified * the last modification time of the file * @return the snapshot. + * @deprecated use {@link #save(Instant)} instead. */ + @Deprecated public static FileSnapshot save(long modified) { - final long read = System.currentTimeMillis(); + final Instant read = Instant.now(); + return new FileSnapshot(read, Instant.ofEpochMilli(modified), + UNKNOWN_SIZE, Duration.ZERO, MISSING_FILEKEY); + } + + /** + * Record a snapshot for a file for which the last modification time is + * already known. + *

+ * This method should be invoked before the file is accessed. + *

+ * Note that this method cannot rely on measuring file timestamp resolution + * to avoid racy git issues caused by finite file timestamp resolution since + * it's unknown in which filesystem the file is located. Hence the worst + * case fallback for timestamp resolution is used. + * + * @param modified + * the last modification time of the file + * @return the snapshot. + */ + public static FileSnapshot save(Instant modified) { + final Instant read = Instant.now(); return new FileSnapshot(read, modified, UNKNOWN_SIZE, Duration.ZERO, MISSING_FILEKEY); } /** Last observed modification time of the path. */ - private final long lastModified; + private final Instant lastModified; /** Last wall-clock time the path was read. */ - private volatile long lastRead; + private volatile Instant lastRead; /** True once {@link #lastRead} is far later than {@link #lastModified}. */ private boolean cannotBeRacilyClean; @@ -222,7 +251,7 @@ protected FileSnapshot(File file) { */ protected FileSnapshot(File file, boolean useConfig) { this.file = file; - this.lastRead = System.currentTimeMillis(); + this.lastRead = Instant.now(); this.fsTimestampResolution = useConfig ? FS.getFsTimerResolution(file.toPath().getParent()) : FALLBACK_TIMESTAMP_RESOLUTION; @@ -230,18 +259,20 @@ protected FileSnapshot(File file, boolean useConfig) { try { fileAttributes = FS.DETECTED.fileAttributes(file); } catch (IOException e) { - this.lastModified = file.lastModified(); + this.lastModified = Instant.ofEpochMilli(file.lastModified()); this.size = file.length(); this.fileKey = MISSING_FILEKEY; return; } - this.lastModified = fileAttributes.lastModifiedTime().toMillis(); + this.lastModified = fileAttributes.lastModifiedTime().toInstant(); this.size = fileAttributes.size(); this.fileKey = getFileKey(fileAttributes); - LOG.debug("file={}, create new FileSnapshot: lastRead={} ms" //$NON-NLS-1$ - + ", lastModified={} ms, size={}, fileKey={}", //$NON-NLS-1$ - file, Long.valueOf(lastRead), Long.valueOf(lastModified), - Long.valueOf(size), fileKey); + if (LOG.isDebugEnabled()) { + LOG.debug("file={}, create new FileSnapshot: lastRead={}, lastModified={}, size={}, fileKey={}", //$NON-NLS-1$ + file, dateFmt.format(lastRead), + dateFmt.format(lastModified), Long.valueOf(size), + fileKey.toString()); + } } private boolean sizeChanged; @@ -252,7 +283,7 @@ protected FileSnapshot(File file, boolean useConfig) { private boolean wasRacyClean; - private FileSnapshot(long read, long modified, long size, + private FileSnapshot(Instant read, Instant modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { this.file = null; this.lastRead = read; @@ -266,8 +297,19 @@ private FileSnapshot(long read, long modified, long size, * Get time of last snapshot update * * @return time of last snapshot update + * @deprecated use {@link #lastModifiedInstant()} instead */ + @Deprecated public long lastModified() { + return lastModified.toEpochMilli(); + } + + /** + * Get time of last snapshot update + * + * @return time of last snapshot update + */ + public Instant lastModifiedInstant() { return lastModified; } @@ -286,16 +328,16 @@ public long size() { * @return true if the path needs to be read again. */ public boolean isModified(File path) { - long currLastModified; + Instant currLastModified; long currSize; Object currFileKey; try { BasicFileAttributes fileAttributes = FS.DETECTED.fileAttributes(path); - currLastModified = fileAttributes.lastModifiedTime().toMillis(); + currLastModified = fileAttributes.lastModifiedTime().toInstant(); currSize = fileAttributes.size(); currFileKey = getFileKey(fileAttributes); } catch (IOException e) { - currLastModified = path.lastModified(); + currLastModified = Instant.ofEpochMilli(path.lastModified()); currSize = path.length(); currFileKey = MISSING_FILEKEY; } @@ -337,7 +379,7 @@ public boolean isModified(File path) { * the other snapshot. */ public void setClean(FileSnapshot other) { - final long now = other.lastRead; + final Instant now = other.lastRead; if (!isRacyClean(now)) { cannotBeRacilyClean = true; } @@ -351,7 +393,7 @@ public void setClean(FileSnapshot other) { * if sleep was interrupted */ public void waitUntilNotRacy() throws InterruptedException { - while (isRacyClean(System.currentTimeMillis())) { + while (isRacyClean(Instant.now())) { TimeUnit.NANOSECONDS .sleep((fsTimestampResolution.toNanos() + 1) * 11 / 10); } @@ -366,7 +408,7 @@ public void waitUntilNotRacy() throws InterruptedException { */ public boolean equals(FileSnapshot other) { boolean sizeEq = size == UNKNOWN_SIZE || other.size == UNKNOWN_SIZE || size == other.size; - return lastModified == other.lastModified && sizeEq + return lastModified.equals(other.lastModified) && sizeEq && Objects.equals(fileKey, other.fileKey); } @@ -389,8 +431,7 @@ public boolean equals(Object obj) { /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(Long.valueOf(lastModified), Long.valueOf(size), - fileKey); + return Objects.hash(lastModified, Long.valueOf(size), fileKey); } /** @@ -435,34 +476,37 @@ public String toString() { if (this == MISSING_FILE) { return "MISSING_FILE"; } - DateFormat f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", - Locale.US); - return "FileSnapshot[modified: " + f.format(new Date(lastModified)) - + ", read: " + f.format(new Date(lastRead)) + ", size:" + size + return "FileSnapshot[modified: " + dateFmt.format(lastModified) + + ", read: " + dateFmt.format(lastRead) + ", size:" + size + ", fileKey: " + fileKey + "]"; } - private boolean isRacyClean(long read) { + private boolean isRacyClean(Instant read) { // add a 10% safety margin long racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; - long delta = (read - lastModified) * 1_000_000; + long delta = Duration.between(lastModified, read).toNanos(); wasRacyClean = delta <= racyNanos; - LOG.debug("file={}, isRacyClean={}, read={} ms, lastModified={} ms," //$NON-NLS-1$ - + " delta={} ns, racy<={} ns", //$NON-NLS-1$ - file, Boolean.valueOf(wasRacyClean), Long.valueOf(read), - Long.valueOf(lastModified), Long.valueOf(delta), - Long.valueOf(racyNanos)); + if (LOG.isDebugEnabled()) { + LOG.debug( + "file={}, isRacyClean={}, read={}, lastModified={}, delta={} ns, racy<={} ns", //$NON-NLS-1$ + file, Boolean.valueOf(wasRacyClean), dateFmt.format(read), + dateFmt.format(lastModified), Long.valueOf(delta), + Long.valueOf(racyNanos)); + } return wasRacyClean; } - private boolean isModified(long currLastModified) { + private boolean isModified(Instant currLastModified) { // Any difference indicates the path was modified. - lastModifiedChanged = lastModified != currLastModified; + lastModifiedChanged = !lastModified.equals(currLastModified); if (lastModifiedChanged) { - LOG.debug("file={}, lastModified changed from {} to {}", //$NON-NLS-1$ - file, Long.valueOf(lastModified), - Long.valueOf(currLastModified)); + if (LOG.isDebugEnabled()) { + LOG.debug( + "file={}, lastModified changed from {} to {}", //$NON-NLS-1$ + file, dateFmt.format(lastModified), + dateFmt.format(currLastModified)); + } return true; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 3c830e88c..791a10828 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -372,8 +372,9 @@ private void deleteOldPacks(Collection oldPacks, continue oldPackLoop; if (!oldPack.shouldBeKept() - && repo.getFS().lastModified( - oldPack.getPackFile()) < packExpireDate) { + && repo.getFS() + .lastModifiedInstant(oldPack.getPackFile()) + .toEpochMilli() < packExpireDate) { oldPack.close(); if (shouldLoosen) { loosen(inserter, reader, oldPack, ids); @@ -561,8 +562,10 @@ public void prune(Set objectsToKeep) throws IOException, String fName = f.getName(); if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) continue; - if (repo.getFS().lastModified(f) >= expireDate) + if (repo.getFS().lastModifiedInstant(f) + .toEpochMilli() >= expireDate) { continue; + } try { ObjectId id = ObjectId.fromString(d + fName); if (objectsToKeep.contains(id)) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java index b80c58ca9..ccca0279a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java @@ -56,8 +56,11 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.FileTime; import java.text.MessageFormat; +import java.time.Instant; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; @@ -424,7 +427,12 @@ public void waitForStatChange() throws InterruptedException { FileSnapshot n = FileSnapshot.save(lck); while (o.equals(n)) { Thread.sleep(25 /* milliseconds */); - lck.setLastModified(System.currentTimeMillis()); + try { + Files.setLastModifiedTime(lck.toPath(), + FileTime.from(Instant.now())); + } catch (IOException e) { + n.waitUntilNotRacy(); + } n = FileSnapshot.save(lck); } } @@ -474,11 +482,22 @@ private void saveStatInformation() { * Get the modification time of the output file when it was committed. * * @return modification time of the lock file right before we committed it. + * @deprecated use {@link #getCommitLastModifiedInstant()} instead */ + @Deprecated public long getCommitLastModified() { return commitSnapshot.lastModified(); } + /** + * Get the modification time of the output file when it was committed. + * + * @return modification time of the lock file right before we committed it. + */ + public Instant getCommitLastModifiedInstant() { + return commitSnapshot.lastModifiedInstant(); + } + /** * Get the {@link FileSnapshot} just before commit. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index 73ad38c95..88e05af41 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -60,6 +60,7 @@ import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; import java.text.MessageFormat; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -107,7 +108,7 @@ public class PackFile implements Iterable { public static final Comparator SORT = new Comparator() { @Override public int compare(PackFile a, PackFile b) { - return b.packLastModified - a.packLastModified; + return b.packLastModified.compareTo(a.packLastModified); } }; @@ -132,7 +133,7 @@ public int compare(PackFile a, PackFile b) { private int activeCopyRawData; - int packLastModified; + Instant packLastModified; private PackFileSnapshot fileSnapshot; @@ -172,7 +173,7 @@ public int compare(PackFile a, PackFile b) { public PackFile(File packFile, int extensions) { this.packFile = packFile; this.fileSnapshot = PackFileSnapshot.save(packFile); - this.packLastModified = (int) (fileSnapshot.lastModified() >> 10); + this.packLastModified = fileSnapshot.lastModifiedInstant(); this.extensions = extensions; // Multiply by 31 here so we can more directly combine with another diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index f60c95f64..d4282e0b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -46,6 +46,7 @@ */ package org.eclipse.jgit.merge; +import static java.time.Instant.EPOCH; import static org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm.HISTOGRAM; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DIFF_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_ALGORITHM; @@ -59,6 +60,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -464,7 +466,7 @@ protected void cleanUp() throws NoWorkTreeException, * @return the entry which was added to the index */ private DirCacheEntry add(byte[] path, CanonicalTreeParser p, int stage, - long lastMod, long len) { + Instant lastMod, long len) { if (p != null && !p.getEntryFileMode().equals(FileMode.TREE)) { DirCacheEntry e = new DirCacheEntry(path, stage); e.setFileMode(p.getEntryFileMode()); @@ -491,7 +493,7 @@ private DirCacheEntry keep(DirCacheEntry e) { e.getStage()); newEntry.setFileMode(e.getFileMode()); newEntry.setObjectId(e.getObjectId()); - newEntry.setLastModified(e.getLastModified()); + newEntry.setLastModified(e.getLastModifiedInstant()); newEntry.setLength(e.getLength()); builder.add(newEntry); return newEntry; @@ -667,16 +669,17 @@ protected boolean processEntry(CanonicalTreeParser base, // we know about length and lastMod only after we have written the new content. // This will happen later. Set these values to 0 for know. DirCacheEntry e = add(tw.getRawPath(), theirs, - DirCacheEntry.STAGE_0, 0, 0); + DirCacheEntry.STAGE_0, EPOCH, 0); addToCheckout(tw.getPathString(), e, attributes); } return true; } else { // FileModes are not mergeable. We found a conflict on modes. // For conflicting entries we don't know lastModified and length. - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, 0, 0); - add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, 0, 0); + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); + add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, + 0); unmergedPaths.add(tw.getPathString()); mergeResults.put( tw.getPathString(), @@ -708,7 +711,7 @@ protected boolean processEntry(CanonicalTreeParser base, // the new content. // This will happen later. Set these values to 0 for know. DirCacheEntry e = add(tw.getRawPath(), theirs, - DirCacheEntry.STAGE_0, 0, 0); + DirCacheEntry.STAGE_0, EPOCH, 0); if (e != null) { addToCheckout(tw.getPathString(), e, attributes); } @@ -737,16 +740,16 @@ protected boolean processEntry(CanonicalTreeParser base, // detected later if (nonTree(modeO) && !nonTree(modeT)) { if (nonTree(modeB)) - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, 0, 0); + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); unmergedPaths.add(tw.getPathString()); enterSubtree = false; return true; } if (nonTree(modeT) && !nonTree(modeO)) { if (nonTree(modeB)) - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); - add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, 0, 0); + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); unmergedPaths.add(tw.getPathString()); enterSubtree = false; return true; @@ -773,9 +776,9 @@ protected boolean processEntry(CanonicalTreeParser base, boolean gitlinkConflict = isGitLink(modeO) || isGitLink(modeT); // Don't attempt to resolve submodule link conflicts if (gitlinkConflict || !attributes.canBeContentMerged()) { - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, 0, 0); - add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, 0, 0); + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); + add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); if (gitlinkConflict) { MergeResult result = new MergeResult<>( @@ -822,10 +825,10 @@ protected boolean processEntry(CanonicalTreeParser base, MergeResult result = contentMerge(base, ours, theirs, attributes); - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, 0, 0); + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); DirCacheEntry e = add(tw.getRawPath(), theirs, - DirCacheEntry.STAGE_3, 0, 0); + DirCacheEntry.STAGE_3, EPOCH, 0); // OURS was deleted checkout THEIRS if (modeO == 0) { @@ -957,9 +960,9 @@ private void updateIndex(CanonicalTreeParser base, // A conflict occurred, the file will contain conflict markers // the index will be populated with the three stages and the // workdir (if used) contains the halfway merged content. - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, 0, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, 0, 0); - add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, 0, 0); + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); + add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); mergeResults.put(tw.getPathString(), result); return; } @@ -976,7 +979,7 @@ private void updateIndex(CanonicalTreeParser base, ? FileMode.REGULAR_FILE : FileMode.fromBits(newMode)); if (mergedFile != null) { dce.setLastModified( - nonNullRepo().getFS().lastModified(mergedFile)); + nonNullRepo().getFS().lastModifiedInstant(mergedFile)); dce.setLength((int) mergedFile.length()); } dce.setObjectId(insertMergeResult(rawMerged, attributes)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/NetRC.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/NetRC.java index e688f6340..4f1eba66d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/NetRC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/NetRC.java @@ -46,6 +46,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.time.Instant; import java.util.Collection; import java.util.HashMap; import java.util.Locale; @@ -123,7 +124,7 @@ boolean complete() { private File netrc; - private long lastModified; + private Instant lastModified; private Map hosts = new HashMap<>(); @@ -187,8 +188,10 @@ public NetRCEntry getEntry(String host) { if (netrc == null) return null; - if (this.lastModified != this.netrc.lastModified()) + if (!this.lastModified + .equals(FS.DETECTED.lastModifiedInstant(this.netrc))) { parse(); + } NetRCEntry entry = this.hosts.get(host); @@ -209,7 +212,7 @@ public Collection getEntries() { private void parse() { this.hosts.clear(); - this.lastModified = this.netrc.lastModified(); + this.lastModified = FS.DETECTED.lastModifiedInstant(this.netrc); try (BufferedReader r = new BufferedReader(new FileReader(netrc))) { String line = null; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java index 480055c2e..4dd5df9cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java @@ -51,6 +51,7 @@ import java.io.InputStreamReader; import java.security.AccessController; import java.security.PrivilegedAction; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -166,7 +167,7 @@ public static OpenSshConfig get(FS fs) { private final File configFile; /** Modification time of {@link #configFile} when it was last loaded. */ - private long lastModified; + private Instant lastModified; /** * Encapsulates entries read out of the configuration file, and @@ -224,8 +225,8 @@ public Host lookup(String hostName) { } private synchronized State refresh() { - final long mtime = configFile.lastModified(); - if (mtime != lastModified) { + final Instant mtime = FS.DETECTED.lastModifiedInstant(configFile); + if (!mtime.equals(lastModified)) { State newState = new State(); try (FileInputStream in = new FileInputStream(configFile)) { newState.entries = parse(in); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java index 3d25c2314..d432c9445 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java @@ -53,6 +53,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -406,8 +407,14 @@ public long getLength() { } @Override + @Deprecated public long getLastModified() { - return attributes.getLastModifiedTime(); + return attributes.getLastModifiedInstant().toEpochMilli(); + } + + @Override + public Instant getLastModifiedInstant() { + return attributes.getLastModifiedInstant(); } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 7c6bfb9d6..299f07fb0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -59,6 +59,7 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CharsetEncoder; import java.text.MessageFormat; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -645,11 +646,23 @@ public long getEntryContentLength() throws IOException { * * @return last modified time of this file, in milliseconds since the epoch * (Jan 1, 1970 UTC). + * @deprecated use {@link #getEntryLastModifiedInstant()} instead */ + @Deprecated public long getEntryLastModified() { return current().getLastModified(); } + /** + * Get the last modified time of this entry. + * + * @return last modified time of this file + * @since 5.1.9 + */ + public Instant getEntryLastModifiedInstant() { + return current().getLastModifiedInstant(); + } + /** * Obtain an input stream to read the file content. *

@@ -924,30 +937,28 @@ public MetadataDiff compareMetadata(DirCacheEntry entry) { // Git under windows only stores seconds so we round the timestamp // Java gives us if it looks like the timestamp in index is seconds - // only. Otherwise we compare the timestamp at millisecond precision, + // only. Otherwise we compare the timestamp at nanosecond precision, // unless core.checkstat is set to "minimal", in which case we only // compare the whole second part. - long cacheLastModified = entry.getLastModified(); - long fileLastModified = getEntryLastModified(); - long lastModifiedMillis = fileLastModified % 1000; - long cacheMillis = cacheLastModified % 1000; - if (getOptions().getCheckStat() == CheckStat.MINIMAL) { - fileLastModified = fileLastModified - lastModifiedMillis; - cacheLastModified = cacheLastModified - cacheMillis; - } else if (cacheMillis == 0) - fileLastModified = fileLastModified - lastModifiedMillis; - // Some Java version on Linux return whole seconds only even when - // the file systems supports more precision. - else if (lastModifiedMillis == 0) - cacheLastModified = cacheLastModified - cacheMillis; - - if (fileLastModified != cacheLastModified) + Instant cacheLastModified = entry.getLastModifiedInstant(); + Instant fileLastModified = getEntryLastModifiedInstant(); + if ((getOptions().getCheckStat() == CheckStat.MINIMAL) + || (cacheLastModified.getNano() == 0) + // Some Java version on Linux return whole seconds only even + // when the file systems supports more precision. + || (fileLastModified.getNano() == 0)) { + if (fileLastModified.getEpochSecond() != cacheLastModified + .getEpochSecond()) { + return MetadataDiff.DIFFER_BY_TIMESTAMP; + } + } + if (!fileLastModified.equals(cacheLastModified)) { return MetadataDiff.DIFFER_BY_TIMESTAMP; - else if (!entry.isSmudged()) - // The file is clean when you look at timestamps. - return MetadataDiff.EQUAL; - else + } else if (entry.isSmudged()) { return MetadataDiff.SMUDGED; + } + // The file is clean when when comparing timestamps + return MetadataDiff.EQUAL; } /** @@ -1274,9 +1285,25 @@ public String toString() { * instance member instead. * * @return time since the epoch (in ms) of the last change. + * @deprecated use {@link #getLastModifiedInstant()} instead */ + @Deprecated public abstract long getLastModified(); + /** + * Get the last modified time of this entry. + *

+ * Note: Efficient implementation required. + *

+ * The implementation of this method must be efficient. If a subclass + * needs to compute the value they should cache the reference within an + * instance member instead. + * + * @return time of the last change. + * @since 5.1.9 + */ + public abstract Instant getLastModifiedInstant(); + /** * Get the name of this entry within its directory. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 6e3e336e9..e8570d741 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.Instant.EPOCH; import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.BufferedReader; @@ -66,6 +67,7 @@ import java.security.PrivilegedAction; import java.text.MessageFormat; import java.time.Duration; +import java.time.Instant; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -625,11 +627,41 @@ public boolean supportsSymlinks() { * @return last modified time of f * @throws java.io.IOException * @since 3.0 + * @deprecated use {@link #lastModifiedInstant(Path)} instead */ + @Deprecated public long lastModified(File f) throws IOException { return FileUtils.lastModified(f); } + /** + * Get the last modified time of a file system object. If the OS/JRE support + * symbolic links, the modification time of the link is returned, rather + * than that of the link target. + * + * @param p + * a {@link Path} object. + * @return last modified time of p + * @since 5.1.9 + */ + public Instant lastModifiedInstant(Path p) { + return FileUtils.lastModifiedInstant(p); + } + + /** + * Get the last modified time of a file system object. If the OS/JRE support + * symbolic links, the modification time of the link is returned, rather + * than that of the link target. + * + * @param f + * a {@link File} object. + * @return last modified time of p + * @since 5.1.9 + */ + public Instant lastModifiedInstant(File f) { + return FileUtils.lastModifiedInstant(f.toPath()); + } + /** * Set the last modified time of a file system object. If the OS/JRE support * symbolic links, the link is modified, not the target, @@ -640,11 +672,28 @@ public long lastModified(File f) throws IOException { * last modified time * @throws java.io.IOException * @since 3.0 + * @deprecated use {@link #setLastModified(Path, Instant)} instead */ + @Deprecated public void setLastModified(File f, long time) throws IOException { FileUtils.setLastModified(f, time); } + /** + * Set the last modified time of a file system object. If the OS/JRE support + * symbolic links, the link is modified, not the target, + * + * @param p + * a {@link Path} object. + * @param time + * last modified time + * @throws java.io.IOException + * @since 5.1.9 + */ + public void setLastModified(Path p, Instant time) throws IOException { + FileUtils.setLastModified(p, time); + } + /** * Get the length of a file or link, If the OS/JRE supports symbolic links * it's the length of the link, else the length of the target. @@ -1712,9 +1761,19 @@ public long getCreationTime() { /** * @return the time (milliseconds since 1970-01-01) when this object was * last modified + * @deprecated use getLastModifiedInstant instead */ + @Deprecated public long getLastModifiedTime() { - return lastModifiedTime; + return lastModifiedInstant.toEpochMilli(); + } + + /** + * @return the time when this object was last modified + * @since 5.1.9 + */ + public Instant getLastModifiedInstant() { + return lastModifiedInstant; } private final boolean isDirectory; @@ -1725,7 +1784,7 @@ public long getLastModifiedTime() { private final long creationTime; - private final long lastModifiedTime; + private final Instant lastModifiedInstant; private final boolean isExecutable; @@ -1743,7 +1802,7 @@ public long getLastModifiedTime() { Attributes(FS fs, File file, boolean exists, boolean isDirectory, boolean isExecutable, boolean isSymbolicLink, boolean isRegularFile, long creationTime, - long lastModifiedTime, long length) { + Instant lastModifiedInstant, long length) { this.fs = fs; this.file = file; this.exists = exists; @@ -1752,7 +1811,7 @@ public long getLastModifiedTime() { this.isSymbolicLink = isSymbolicLink; this.isRegularFile = isRegularFile; this.creationTime = creationTime; - this.lastModifiedTime = lastModifiedTime; + this.lastModifiedInstant = lastModifiedInstant; this.length = length; } @@ -1764,7 +1823,7 @@ public long getLastModifiedTime() { * @param path */ public Attributes(File path, FS fs) { - this(fs, path, false, false, false, false, false, 0L, 0L, 0L); + this(fs, path, false, false, false, false, false, 0L, EPOCH, 0L); } /** @@ -1810,7 +1869,7 @@ public Attributes getAttributes(File path) { boolean exists = isDirectory || isFile; boolean canExecute = exists && !isDirectory && canExecute(path); boolean isSymlink = false; - long lastModified = exists ? path.lastModified() : 0L; + Instant lastModified = exists ? lastModifiedInstant(path) : EPOCH; long createTime = 0L; return new Attributes(this, path, exists, isDirectory, canExecute, isSymlink, isFile, createTime, lastModified, -1); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java index 98797dc64..7c0727036 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java @@ -149,7 +149,7 @@ public FileVisitResult visitFile(Path file, attrs.isSymbolicLink(), attrs.isRegularFile(), attrs.creationTime().toMillis(), - attrs.lastModifiedTime().toMillis(), + attrs.lastModifiedTime().toInstant(), attrs.size()); result.add(new FileEntry(f, fs, attributes, fileModeStrategy)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index 9650602fe..80f188cb2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -76,11 +76,14 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.util.FS.Attributes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * File Utilities */ public class FileUtils { + private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class); /** * Option to delete given {@code File} @@ -656,12 +659,31 @@ static boolean isSymlink(File file) { * @return lastModified attribute for given file, not following symbolic * links * @throws IOException + * @deprecated use {@link #lastModifiedInstant(Path)} instead which returns + * FileTime */ + @Deprecated static long lastModified(File file) throws IOException { return Files.getLastModifiedTime(toPath(file), LinkOption.NOFOLLOW_LINKS) .toMillis(); } + /** + * @param path + * @return lastModified attribute for given file, not following symbolic + * links + */ + static Instant lastModifiedInstant(Path path) { + try { + return Files.getLastModifiedTime(path, LinkOption.NOFOLLOW_LINKS) + .toInstant(); + } catch (IOException e) { + LOG.error(MessageFormat + .format(JGitText.get().readLastModifiedFailed, path)); + return Instant.ofEpochMilli(path.toFile().lastModified()); + } + } + /** * Return all the attributes of a file, without following symbolic links. * @@ -680,10 +702,21 @@ static BasicFileAttributes fileAttributes(File file) throws IOException { * @param time * @throws IOException */ + @Deprecated static void setLastModified(File file, long time) throws IOException { Files.setLastModifiedTime(toPath(file), FileTime.fromMillis(time)); } + /** + * @param path + * @param time + * @throws IOException + */ + static void setLastModified(Path path, Instant time) + throws IOException { + Files.setLastModifiedTime(path, FileTime.from(time)); + } + /** * @param file * @return {@code true} if the given file exists, not following symbolic @@ -788,7 +821,7 @@ static Attributes getFileAttributesBasic(FS fs, File file) { readAttributes.isSymbolicLink(), readAttributes.isRegularFile(), // readAttributes.creationTime().toMillis(), // - readAttributes.lastModifiedTime().toMillis(), + readAttributes.lastModifiedTime().toInstant(), readAttributes.isSymbolicLink() ? Constants .encode(readSymLink(file)).length : readAttributes.size()); @@ -827,7 +860,7 @@ public static Attributes getFileAttributesPosix(FS fs, File file) { readAttributes.isSymbolicLink(), readAttributes.isRegularFile(), // readAttributes.creationTime().toMillis(), // - readAttributes.lastModifiedTime().toMillis(), + readAttributes.lastModifiedTime().toInstant(), readAttributes.size()); return attributes; } catch (IOException e) { From d8d94272770352515c129ad6d980f2b6e8179409 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 18 Jul 2019 03:36:18 +0200 Subject: [PATCH 26/47] Use Instant for smudge time in DirCache and DirCacheEntry Change-Id: I98050a51baf4726c5717ef62ce7f026173666bdf Signed-off-by: Matthias Sohn --- org.eclipse.jgit/.settings/.api_filters | 6 ++++ .../org/eclipse/jgit/dircache/DirCache.java | 10 +++--- .../eclipse/jgit/dircache/DirCacheEntry.java | 34 +++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 951a5e30f..01215a8c7 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -15,6 +15,12 @@ + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java index 340214b06..0cfd16b58 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java @@ -58,6 +58,7 @@ import java.security.DigestOutputStream; import java.security.MessageDigest; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -497,9 +498,7 @@ else if (ver != 2) throw new CorruptObjectException(JGitText.get().DIRCHasTooManyEntries); snapshot = FileSnapshot.save(liveFile); - // TODO (ms) combine smudge_s and smudge_ns into Duration - int smudge_s = (int) (snapshot.lastModifiedInstant().getEpochSecond()); - int smudge_ns = snapshot.lastModifiedInstant().getNano(); + Instant smudge = snapshot.lastModifiedInstant(); // Load the individual file entries. // @@ -508,8 +507,9 @@ else if (ver != 2) sortedEntries = new DirCacheEntry[entryCnt]; final MutableInteger infoAt = new MutableInteger(); - for (int i = 0; i < entryCnt; i++) - sortedEntries[i] = new DirCacheEntry(infos, infoAt, in, md, smudge_s, smudge_ns); + for (int i = 0; i < entryCnt; i++) { + sortedEntries[i] = new DirCacheEntry(infos, infoAt, in, md, smudge); + } // After the file entries are index extensions, and then a footer. // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java index 4b36f6b96..d2a59c131 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java @@ -145,10 +145,9 @@ public class DirCacheEntry { /** Flags which are never stored to disk. */ private byte inCoreFlags; - // TODO (ms): use Instant to combine smudge_s and smudge_ns DirCacheEntry(final byte[] sharedInfo, final MutableInteger infoAt, - final InputStream in, final MessageDigest md, final int smudge_s, - final int smudge_ns) throws IOException { + final InputStream in, final MessageDigest md, final Instant smudge) + throws IOException { info = sharedInfo; infoOffset = infoAt.value; @@ -217,8 +216,9 @@ public class DirCacheEntry { md.update(nullpad, 0, padLen); } - if (mightBeRacilyClean(smudge_s, smudge_ns)) + if (mightBeRacilyClean(smudge)) { smudgeRacilyClean(); + } } /** @@ -346,8 +346,29 @@ void write(OutputStream os) throws IOException { * @param smudge_ns * nanoseconds component of the index's last modified time. * @return true if extra careful checks should be used. + * @deprecated use {@link #mightBeRacilyClean(Instant)} instead */ + @Deprecated public final boolean mightBeRacilyClean(int smudge_s, int smudge_ns) { + return mightBeRacilyClean(Instant.ofEpochSecond(smudge_s, smudge_ns)); + } + + /** + * Is it possible for this entry to be accidentally assumed clean? + *

+ * The "racy git" problem happens when a work file can be updated faster + * than the filesystem records file modification timestamps. It is possible + * for an application to edit a work file, update the index, then edit it + * again before the filesystem will give the work file a new modification + * timestamp. This method tests to see if file was written out at the same + * time as the index. + * + * @param smudge + * index's last modified time. + * @return true if extra careful checks should be used. + * @since 5.1.9 + */ + public final boolean mightBeRacilyClean(Instant smudge) { // If the index has a modification time then it came from disk // and was not generated from scratch in memory. In such cases // the entry is 'racily clean' if the entry's cached modification @@ -357,8 +378,9 @@ public final boolean mightBeRacilyClean(int smudge_s, int smudge_ns) { // final int base = infoOffset + P_MTIME; final int mtime = NB.decodeInt32(info, base); - if (smudge_s == mtime) - return smudge_ns <= NB.decodeInt32(info, base + 4); + if (smudge.getEpochSecond() == mtime) { + return smudge.getNano() <= NB.decodeInt32(info, base + 4); + } return false; } From 72d6e304b85a310d3ad271e61005eb22190b3f44 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 19 Jul 2019 14:42:40 +0200 Subject: [PATCH 27/47] Fix FileSnapshot#saveNoConfig We should not use configuration when creating FileSnapshot when accessing FileBasedConfig. Change-Id: Ic521632870f18bb004751642b9d30648dd94049a Signed-off-by: Matthias Sohn --- .../org/eclipse/jgit/internal/storage/file/FileSnapshot.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index fe3703719..ec71783c6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -147,7 +147,7 @@ public static FileSnapshot save(File path) { * @return the snapshot. */ public static FileSnapshot saveNoConfig(File path) { - return new FileSnapshot(path); + return new FileSnapshot(path, false); } private static Object getFileKey(BasicFileAttributes fileAttributes) { From 37f7679fc956729a5e06509593c7cfc0d5801a2c Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 19 Jul 2019 14:43:52 +0200 Subject: [PATCH 28/47] Handle CancellationException in FileStoreAttributeCache Change-Id: If5985fbf04f630b1d72a1bafd508e0e15e1436be --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index e8570d741..7a03593ab 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -74,6 +74,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -281,7 +282,7 @@ private static Duration getFsTimestampResolution(Path file) { } // return fallback until measurement is finished } catch (IOException | InterruptedException - | ExecutionException e) { + | ExecutionException | CancellationException e) { LOG.error(e.getMessage(), e); } catch (TimeoutException | SecurityException e) { // use fallback From 99d351d0cbcf93502a5a6888c4032b8bb1fc2e92 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 8 Jul 2019 10:20:16 +0200 Subject: [PATCH 29/47] Measure stored timestamp resolution instead of time to touch file Measure granularity of timestamps stored in the filesystem by setting and then getting lastModified timestamp until the read value changed. Increase increment exponentially to limit number of iterations starting with 1 microsecond since Java's FileTime (up to Java 12) truncates timestamps to 1 microsecond resolution. The chosen algorithm yields 2000 steps between 1 ms and 2.5 s. Also measure clock resolution and add that for the total timestamp resolution. This avoids systematic measurement errors introduced by doing IO to touch a file. Change-Id: I9b37138619422452373e298d9d8c7cb2c384db3f Signed-off-by: Matthias Sohn --- .../src/org/eclipse/jgit/util/FS.java | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 7a03593ab..e33c4bff9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -299,46 +299,45 @@ private static Optional measureFsTimestampResolution( Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ try { Files.createFile(probe); - // ensure we always use the local system clock - FileUtils.touch(probe); - long wait = 512; - long start = System.nanoTime(); FileTime t1 = Files.getLastModifiedTime(probe); FileTime t2 = t1; - while (t2.compareTo(t1) <= 0) { - TimeUnit.NANOSECONDS.sleep(wait); - checkTimeout(s, start); - FileUtils.touch(probe); + Instant t1i = t1.toInstant(); + for (long i = 1; t2.compareTo(t1) <= 0; i += 1 + i / 20) { + Files.setLastModifiedTime(probe, + FileTime.from(t1i.plusNanos(i * 1000))); t2 = Files.getLastModifiedTime(probe); - if (wait < 100_000_000L) { - wait = wait * 2; - } } - Duration resolution = Duration.between(t1.toInstant(), t2.toInstant()); - saveFileTimeResolution(s, resolution); - return Optional.of(resolution); + Duration fsResolution = Duration.between(t1.toInstant(), t2.toInstant()); + Duration clockResolution = measureClockResolution(); + fsResolution = fsResolution.plus(clockResolution); + saveFileTimeResolution(s, fsResolution); + return Optional.of(fsResolution); } catch (AccessDeniedException e) { LOG.warn(e.getLocalizedMessage(), e); // see bug 548648 - } catch (IOException | TimeoutException e) { + } catch (IOException e) { LOG.error(e.getLocalizedMessage(), e); - } catch (InterruptedException e) { - LOG.error(e.getLocalizedMessage(), e); - Thread.currentThread().interrupt(); } finally { deleteProbe(probe); } return Optional.empty(); } - private static void checkTimeout(FileStore s, long start) - throws TimeoutException { - if (System.nanoTime() - start >= FALLBACK_TIMESTAMP_RESOLUTION - .toNanos()) { - throw new TimeoutException(MessageFormat.format(JGitText - .get().timeoutMeasureFsTimestampResolution, - s.toString())); + private static Duration measureClockResolution() { + Duration clockResolution = Duration.ZERO; + for (int i = 0; i < 10; i++) { + Instant t1 = Instant.now(); + Instant t2 = t1; + while (t2.compareTo(t1) <= 0) { + t2 = Instant.now(); + } + Duration r = Duration.between(t1, t2); + if (r.compareTo(clockResolution) > 0) { + clockResolution = r; + } } + return clockResolution; } + private static void deleteProbe(Path probe) { if (Files.exists(probe)) { try { From 93144f1438b58264b9fe0dfed195407bc9c0ab1b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 10 Jul 2019 14:25:20 +0200 Subject: [PATCH 30/47] Refactor FileSnapshotTest to use NIO APIs - use Path instead of File - create test directories, files and output stream using Files methods - delete unused list "files" Change-Id: I8c5c601eca9f613efb5618d33b262277df92a06a Signed-off-by: Matthias Sohn --- .../storage/file/FileSnapshotTest.java | 77 ++++++------- .../storage/file/FileBasedConfigTest.java | 105 ++++++++++-------- 2 files changed, 93 insertions(+), 89 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 467f6956b..3031cb944 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -45,15 +45,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.util.ArrayList; -import java.util.List; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; @@ -65,27 +64,24 @@ public class FileSnapshotTest { - private List files = new ArrayList<>(); - - private File trash; + private Path trash; @Before public void setUp() throws Exception { - trash = File.createTempFile("tmp_", ""); - trash.delete(); - assertTrue("mkdir " + trash, trash.mkdir()); + trash = Files.createTempDirectory("tmp_"); } @Before @After public void tearDown() throws Exception { - FileUtils.delete(trash, FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); + FileUtils.delete(trash.toFile(), + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); } - private static void waitNextTick(File f) throws IOException { + private static void waitNextTick(Path f) throws IOException { Instant initialLastModified = FS.DETECTED.lastModifiedInstant(f); do { - FS.DETECTED.setLastModified(f.toPath(), Instant.now()); + FS.DETECTED.setLastModified(f, Instant.now()); } while (FS.DETECTED.lastModifiedInstant(f) .equals(initialLastModified)); } @@ -97,12 +93,12 @@ private static void waitNextTick(File f) throws IOException { */ @Test public void testActuallyIsModifiedTrivial() throws Exception { - File f1 = createFile("simple"); + Path f1 = createFile("simple"); waitNextTick(f1); - FileSnapshot save = FileSnapshot.save(f1); + FileSnapshot save = FileSnapshot.save(f1.toFile()); append(f1, (byte) 'x'); waitNextTick(f1); - assertTrue(save.isModified(f1)); + assertTrue(save.isModified(f1.toFile())); } /** @@ -115,11 +111,11 @@ public void testActuallyIsModifiedTrivial() throws Exception { */ @Test public void testNewFileWithWait() throws Exception { - File f1 = createFile("newfile"); + Path f1 = createFile("newfile"); waitNextTick(f1); - FileSnapshot save = FileSnapshot.save(f1); + FileSnapshot save = FileSnapshot.save(f1.toFile()); Thread.sleep(1500); - assertTrue(save.isModified(f1)); + assertTrue(save.isModified(f1.toFile())); } /** @@ -129,9 +125,9 @@ public void testNewFileWithWait() throws Exception { */ @Test public void testNewFileNoWait() throws Exception { - File f1 = createFile("newfile"); - FileSnapshot save = FileSnapshot.save(f1); - assertTrue(save.isModified(f1)); + Path f1 = createFile("newfile"); + FileSnapshot save = FileSnapshot.save(f1.toFile()); + assertTrue(save.isModified(f1.toFile())); } /** @@ -145,19 +141,19 @@ public void testNewFileNoWait() throws Exception { @Test public void testSimulatePackfileReplacement() throws Exception { Assume.assumeFalse(SystemReader.getInstance().isWindows()); - File f1 = createFile("file"); // inode y - File f2 = createFile("fool"); // Guarantees new inode x + Path f1 = createFile("file"); // inode y + Path f2 = createFile("fool"); // Guarantees new inode x // wait on f2 since this method resets lastModified of the file // and leaves lastModified of f1 untouched waitNextTick(f2); waitNextTick(f2); - FileTime timestamp = Files.getLastModifiedTime(f1.toPath()); - FileSnapshot save = FileSnapshot.save(f1); - Files.move(f2.toPath(), f1.toPath(), // Now "file" is inode x + FileTime timestamp = Files.getLastModifiedTime(f1); + FileSnapshot save = FileSnapshot.save(f1.toFile()); + Files.move(f2, f1, // Now "file" is inode x StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - Files.setLastModifiedTime(f1.toPath(), timestamp); - assertTrue(save.isModified(f1)); + Files.setLastModifiedTime(f1, timestamp); + assertTrue(save.isModified(f1.toFile())); assertTrue("unexpected change of fileKey", save.wasFileKeyChanged()); assertFalse("unexpected size change", save.wasSizeChanged()); assertFalse("unexpected lastModified change", @@ -174,12 +170,12 @@ public void testSimulatePackfileReplacement() throws Exception { */ @Test public void testFileSizeChanged() throws Exception { - File f = createFile("file"); - FileTime timestamp = Files.getLastModifiedTime(f.toPath()); - FileSnapshot save = FileSnapshot.save(f); + Path f = createFile("file"); + FileTime timestamp = Files.getLastModifiedTime(f); + FileSnapshot save = FileSnapshot.save(f.toFile()); append(f, (byte) 'x'); - Files.setLastModifiedTime(f.toPath(), timestamp); - assertTrue(save.isModified(f)); + Files.setLastModifiedTime(f, timestamp); + assertTrue(save.isModified(f.toFile())); assertTrue(save.wasSizeChanged()); } @@ -194,15 +190,14 @@ public void fileSnapshotEquals() throws Exception { assertTrue(fs2.equals(fs1)); } - private File createFile(String string) throws IOException { - trash.mkdirs(); - File f = File.createTempFile(string, "tdat", trash); - files.add(f); - return f; + private Path createFile(String string) throws IOException { + Files.createDirectories(trash); + return Files.createTempFile(trash, string, "tdat"); } - private static void append(File f, byte b) throws IOException { - try (FileOutputStream os = new FileOutputStream(f, true)) { + private static void append(Path f, byte b) throws IOException { + try (OutputStream os = Files.newOutputStream(f, + StandardOpenOption.APPEND)) { os.write(b); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index 7f0d60295..287ad320f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -46,12 +46,13 @@ import static org.eclipse.jgit.util.FileUtils.pathToString; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.util.FS; @@ -77,42 +78,43 @@ public class FileBasedConfigTest { private static final String CONTENT2 = "[" + USER + "]\n\t" + NAME + " = " + BOB + "\n"; - private File trash; + private Path trash; @Before public void setUp() throws Exception { - trash = File.createTempFile("tmp_", ""); - trash.delete(); - assertTrue("mkdir " + trash, trash.mkdir()); + trash = Files.createTempDirectory("tmp_"); } @After public void tearDown() throws Exception { - FileUtils.delete(trash, FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); + FileUtils.delete(trash.toFile(), + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); } @Test public void testSystemEncoding() throws IOException, ConfigInvalidException { - final File file = createFile(CONTENT1.getBytes()); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(CONTENT1.getBytes()); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); config.setString(USER, null, NAME, BOB); config.save(); - assertArrayEquals(CONTENT2.getBytes(), IO.readFully(file)); + assertArrayEquals(CONTENT2.getBytes(), IO.readFully(file.toFile())); } @Test public void testUTF8withoutBOM() throws IOException, ConfigInvalidException { - final File file = createFile(CONTENT1.getBytes(UTF_8)); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(CONTENT1.getBytes(UTF_8)); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); config.setString(USER, null, NAME, BOB); config.save(); - assertArrayEquals(CONTENT2.getBytes(), IO.readFully(file)); + assertArrayEquals(CONTENT2.getBytes(), IO.readFully(file.toFile())); } @Test @@ -123,8 +125,9 @@ public void testUTF8withBOM() throws IOException, ConfigInvalidException { bos1.write(0xBF); bos1.write(CONTENT1.getBytes(UTF_8)); - final File file = createFile(bos1.toByteArray()); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(bos1.toByteArray()); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); @@ -136,7 +139,7 @@ public void testUTF8withBOM() throws IOException, ConfigInvalidException { bos2.write(0xBB); bos2.write(0xBF); bos2.write(CONTENT2.getBytes(UTF_8)); - assertArrayEquals(bos2.toByteArray(), IO.readFully(file)); + assertArrayEquals(bos2.toByteArray(), IO.readFully(file.toFile())); } @Test @@ -145,8 +148,9 @@ public void testLeadingWhitespaces() throws IOException, ConfigInvalidException bos1.write(" \n\t".getBytes()); bos1.write(CONTENT1.getBytes()); - final File file = createFile(bos1.toByteArray()); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(bos1.toByteArray()); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); @@ -156,19 +160,20 @@ public void testLeadingWhitespaces() throws IOException, ConfigInvalidException final ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); bos2.write(" \n\t".getBytes()); bos2.write(CONTENT2.getBytes()); - assertArrayEquals(bos2.toByteArray(), IO.readFully(file)); + assertArrayEquals(bos2.toByteArray(), IO.readFully(file.toFile())); } @Test public void testIncludeAbsolute() throws IOException, ConfigInvalidException { - final File includedFile = createFile(CONTENT1.getBytes()); + final Path includedFile = createFile(CONTENT1.getBytes()); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write("[include]\npath=".getBytes()); - bos.write(pathToString(includedFile).getBytes()); + bos.write(pathToString(includedFile.toFile()).getBytes()); - final File file = createFile(bos.toByteArray()); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(bos.toByteArray()); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); } @@ -176,13 +181,14 @@ public void testIncludeAbsolute() @Test public void testIncludeRelativeDot() throws IOException, ConfigInvalidException { - final File includedFile = createFile(CONTENT1.getBytes(), "dir1"); + final Path includedFile = createFile(CONTENT1.getBytes(), "dir1"); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write("[include]\npath=".getBytes()); - bos.write(("./" + includedFile.getName()).getBytes()); + bos.write(("./" + includedFile.getFileName()).getBytes()); - final File file = createFile(bos.toByteArray(), "dir1"); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(bos.toByteArray(), "dir1"); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); } @@ -190,14 +196,15 @@ public void testIncludeRelativeDot() @Test public void testIncludeRelativeDotDot() throws IOException, ConfigInvalidException { - final File includedFile = createFile(CONTENT1.getBytes(), "dir1"); + final Path includedFile = createFile(CONTENT1.getBytes(), "dir1"); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write("[include]\npath=".getBytes()); - bos.write(("../" + includedFile.getParentFile().getName() + "/" - + includedFile.getName()).getBytes()); + bos.write(("../" + includedFile.getParent().getFileName() + "/" + + includedFile.getFileName()).getBytes()); - final File file = createFile(bos.toByteArray(), "dir2"); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(bos.toByteArray(), "dir2"); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); } @@ -205,13 +212,14 @@ public void testIncludeRelativeDotDot() @Test public void testIncludeRelativeDotDotNotFound() throws IOException, ConfigInvalidException { - final File includedFile = createFile(CONTENT1.getBytes()); + final Path includedFile = createFile(CONTENT1.getBytes()); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write("[include]\npath=".getBytes()); - bos.write(("../" + includedFile.getName()).getBytes()); + bos.write(("../" + includedFile.getFileName()).getBytes()); - final File file = createFile(bos.toByteArray()); - final FileBasedConfig config = new FileBasedConfig(file, FS.DETECTED); + final Path file = createFile(bos.toByteArray()); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), + FS.DETECTED); config.load(); assertEquals(null, config.getString(USER, null, NAME)); } @@ -219,30 +227,31 @@ public void testIncludeRelativeDotDotNotFound() @Test public void testIncludeWithTilde() throws IOException, ConfigInvalidException { - final File includedFile = createFile(CONTENT1.getBytes(), "home"); + final Path includedFile = createFile(CONTENT1.getBytes(), "home"); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write("[include]\npath=".getBytes()); - bos.write(("~/" + includedFile.getName()).getBytes()); + bos.write(("~/" + includedFile.getFileName()).getBytes()); - final File file = createFile(bos.toByteArray(), "repo"); + final Path file = createFile(bos.toByteArray(), "repo"); final FS fs = FS.DETECTED.newInstance(); - fs.setUserHome(includedFile.getParentFile()); + fs.setUserHome(includedFile.getParent().toFile()); - final FileBasedConfig config = new FileBasedConfig(file, fs); + final FileBasedConfig config = new FileBasedConfig(file.toFile(), fs); config.load(); assertEquals(ALICE, config.getString(USER, null, NAME)); } - private File createFile(byte[] content) throws IOException { + private Path createFile(byte[] content) throws IOException { return createFile(content, null); } - private File createFile(byte[] content, String subdir) throws IOException { - File dir = subdir != null ? new File(trash, subdir) : trash; - dir.mkdirs(); + private Path createFile(byte[] content, String subdir) throws IOException { + Path dir = subdir != null ? trash.resolve(subdir) : trash; + Files.createDirectories(dir); - File f = File.createTempFile(getClass().getName(), null, dir); - try (FileOutputStream os = new FileOutputStream(f, true)) { + Path f = Files.createTempFile(dir, getClass().getName(), null); + try (OutputStream os = Files.newOutputStream(f, + StandardOpenOption.APPEND)) { os.write(content); } return f; From eda2e95fa8748813cbcfc7b7fbedefb002999cfa Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 10 Jul 2019 16:22:15 +0200 Subject: [PATCH 31/47] Measure filesystem timestamp resolution already in test setup This helps to avoid some time critical tests can't prepare the test fixture intended since measuring timestamp resolution takes time. Change-Id: Ib34023e682a106070ca97e98ef16789a4dfb97b4 Signed-off-by: Matthias Sohn --- .../src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java | 3 ++- .../org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java | 4 ++++ .../tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java | 6 ++++++ .../jgit/internal/storage/file/FileSnapshotTest.java | 3 +++ .../org/eclipse/jgit/storage/file/FileBasedConfigTest.java | 1 + 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java index 3ce066376..1d7187a31 100644 --- a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java +++ b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java @@ -65,12 +65,13 @@ public class GitCloneTaskTest extends LocalDiskRepositoryTestCase { @Before public void before() throws IOException { + dest = createTempFile(); + FS.getFsTimerResolution(dest.toPath().getParent()); project = new Project(); project.init(); enableLogging(); project.addTaskDefinition("git-clone", GitCloneTask.class); task = (GitCloneTask) project.createTask("git-clone"); - dest = createTempFile(); task.setDest(dest); } diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index 50e6f2da5..af688d234 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -127,6 +127,10 @@ public void setUp() throws Exception { if (!tmp.delete() || !tmp.mkdir()) throw new IOException("Cannot create " + tmp); + // measure timer resolution before the test to avoid time critical tests + // are affected by time needed for measurement + FS.getFsTimerResolution(tmp.toPath().getParent()); + mockSystemReader = new MockSystemReader(); mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, "usergitconfig"), FS.DETECTED); diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 10823b878..63af6eb52 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -82,6 +82,7 @@ import org.eclipse.jgit.lfs.server.LargeFileRepository; import org.eclipse.jgit.lfs.server.LfsProtocolServlet; import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; import org.junit.After; @@ -119,6 +120,11 @@ public Path getDir() { @Before public void setup() throws Exception { tmp = Files.createTempDirectory("jgit_test_"); + + // measure timer resolution before the test to avoid time critical tests + // are affected by time needed for measurement + FS.getFsTimerResolution(tmp.getParent()); + server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); dir = Paths.get(tmp.toString(), "lfs"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 3031cb944..f228fb330 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -69,6 +69,9 @@ public class FileSnapshotTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); + // measure timer resolution before the test to avoid time critical tests + // are affected by time needed for measurement + FS.getFsTimerResolution(trash.getParent()); } @Before diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index 287ad320f..20a76704c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -83,6 +83,7 @@ public class FileBasedConfigTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); + FS.getFsTimerResolution(trash.getParent()); } @After From bce4ac97fabc5ea3627c06e24f7a2fab38642514 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 17 Jul 2019 10:11:05 +0200 Subject: [PATCH 32/47] Retry deleting test files in FileBasedConfigTest Change-Id: I304b2b6f2e39f72f620bba53aead60256aed3660 Signed-off-by: Matthias Sohn --- .../tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index 20a76704c..d3686285e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -89,7 +89,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { FileUtils.delete(trash.toFile(), - FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING | FileUtils.RETRY); } @Test From 9eff45e4f2e54a08a04b91aae47b5245f74c0582 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 10 Jul 2019 16:17:21 +0200 Subject: [PATCH 33/47] Fix FileSnapshotTests for filesystem with high timestamp resolution When filesystem timestamp resolution is very high some tests don't work since runtime of the test setup is too long to reach a racily clean FileSnapshot. Hence skip these tests when timestamp resolution is higher than 10 millisecond. Change-Id: Ie47dd10eda22037b5c1ebff6b6becce0654ea807 Signed-off-by: Matthias Sohn --- .../internal/storage/file/FileSnapshotTest.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index f228fb330..1dea47084 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -52,7 +52,9 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileTime; +import java.time.Duration; import java.time.Instant; +import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; @@ -66,12 +68,14 @@ public class FileSnapshotTest { private Path trash; + private Duration fsTimerResolution; + @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(trash.getParent()); + fsTimerResolution = FS.getFsTimerResolution(trash.getParent()); } @Before @@ -114,10 +118,14 @@ public void testActuallyIsModifiedTrivial() throws Exception { */ @Test public void testNewFileWithWait() throws Exception { + // if filesystem timestamp resolution is high the snapshot won't be + // racily clean + Assume.assumeTrue( + fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); waitNextTick(f1); FileSnapshot save = FileSnapshot.save(f1.toFile()); - Thread.sleep(1500); + TimeUnit.NANOSECONDS.sleep(fsTimerResolution.dividedBy(2).toNanos()); assertTrue(save.isModified(f1.toFile())); } @@ -128,6 +136,10 @@ public void testNewFileWithWait() throws Exception { */ @Test public void testNewFileNoWait() throws Exception { + // if filesystem timestamp resolution is high the snapshot won't be + // racily clean + Assume.assumeTrue( + fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); FileSnapshot save = FileSnapshot.save(f1.toFile()); assertTrue(save.isModified(f1.toFile())); From ba5d13ed42d0b312a298b2197e638fb280441a51 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 12 Jul 2019 09:45:09 +0200 Subject: [PATCH 34/47] Enhance RepeatRule to report number of failures at the end In order to enable counting how frequently a test fails if repeated add option abortOnFailure. If it is true the test aborts on the first failure. Otherwise it runs the configured number of repetitions and, if there was any failure, throws a RepeatException reporting how many of the test repetitions failed. Change-Id: Ic47de44d4a6273fddf04b9993ad989903efb40c3 Signed-off-by: Matthias Sohn --- .../src/org/eclipse/jgit/junit/Repeat.java | 9 ++++++ .../org/eclipse/jgit/junit/RepeatRule.java | 28 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java index 08220ce24..94df554ae 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java @@ -56,4 +56,13 @@ * Number of repetitions */ public abstract int n(); + + /** + * Whether to abort execution on first test failure + * + * @return {@code true} if execution should be aborted on the first failure, + * otherwise count failures and continue execution + * @since 5.1.9 + */ + public boolean abortOnFailure() default true; } diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java index 8165738ed..a76f34853 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java @@ -84,6 +84,10 @@ public class RepeatRule implements TestRule { public static class RepeatedTestException extends RuntimeException { private static final long serialVersionUID = 1L; + public RepeatedTestException(String message) { + super(message); + } + public RepeatedTestException(String message, Throwable cause) { super(message, cause); } @@ -93,28 +97,45 @@ private static class RepeatStatement extends Statement { private final int repetitions; + private boolean abortOnFailure; + private final Statement statement; - private RepeatStatement(int repetitions, Statement statement) { + private RepeatStatement(int repetitions, boolean abortOnFailure, + Statement statement) { this.repetitions = repetitions; + this.abortOnFailure = abortOnFailure; this.statement = statement; } @Override public void evaluate() throws Throwable { + int failures = 0; for (int i = 0; i < repetitions; i++) { try { statement.evaluate(); } catch (Throwable e) { + failures += 1; RepeatedTestException ex = new RepeatedTestException( MessageFormat.format( "Repeated test failed when run for the {0}. time", Integer.valueOf(i + 1)), e); LOG.log(Level.SEVERE, ex.getMessage(), ex); - throw ex; + if (abortOnFailure) { + throw ex; + } } } + if (failures > 0) { + RepeatedTestException e = new RepeatedTestException( + MessageFormat.format( + "Test failed {0} times out of {1} repeated executions", + Integer.valueOf(failures), + Integer.valueOf(repetitions))); + LOG.log(Level.SEVERE, e.getMessage(), e); + throw e; + } } } @@ -125,7 +146,8 @@ public Statement apply(Statement statement, Description description) { Repeat repeat = description.getAnnotation(Repeat.class); if (repeat != null) { int n = repeat.n(); - result = new RepeatStatement(n, statement); + boolean abortOnFailure = repeat.abortOnFailure(); + result = new RepeatStatement(n, abortOnFailure, statement); } return result; } From 376c20f45457c0db11850c6e45e11ee764072508 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 18 Jul 2019 13:25:39 +0200 Subject: [PATCH 35/47] Add missing javadoc in org.eclipse.jgit.junit Change-Id: Ib709ef050bec31c87f542fb2cc977863dda93ef9 Signed-off-by: Matthias Sohn --- .../org/eclipse/jgit/junit/RepeatRule.java | 18 +++ .../eclipse/jgit/junit/TestRepository.java | 139 ++++++++++++++++++ 2 files changed, 157 insertions(+) diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java index a76f34853..8636f2a0a 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java @@ -81,13 +81,31 @@ public class RepeatRule implements TestRule { private static Logger LOG = Logger .getLogger(RepeatRule.class.getName()); + /** + * Exception thrown if repeated execution of a test annotated with + * {@code @Repeat} failed. + */ public static class RepeatedTestException extends RuntimeException { private static final long serialVersionUID = 1L; + /** + * Constructor + * + * @param message + * the error message + */ public RepeatedTestException(String message) { super(message); } + /** + * Constructor + * + * @param message + * the error message + * @param cause + * exception causing this exception + */ public RepeatedTestException(String message, Throwable cause) { super(message, cause); } diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index c9fa2f506..49a2b2010 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java @@ -1059,6 +1059,14 @@ public class CommitBuilder { parents.add(prior.create()); } + /** + * set parent commit + * + * @param p + * parent commit + * @return this commit builder + * @throws Exception + */ public CommitBuilder parent(RevCommit p) throws Exception { if (parents.isEmpty()) { DirCacheBuilder b = tree.builder(); @@ -1071,29 +1079,71 @@ public CommitBuilder parent(RevCommit p) throws Exception { return this; } + /** + * Get parent commits + * + * @return parent commits + */ public List parents() { return Collections.unmodifiableList(parents); } + /** + * Remove parent commits + * + * @return this commit builder + */ public CommitBuilder noParents() { parents.clear(); return this; } + /** + * Remove files + * + * @return this commit builder + */ public CommitBuilder noFiles() { tree.clear(); return this; } + /** + * Set top level tree + * + * @param treeId + * the top level tree + * @return this commit builder + */ public CommitBuilder setTopLevelTree(ObjectId treeId) { topLevelTree = treeId; return this; } + /** + * Add file with given content + * + * @param path + * path of the file + * @param content + * the file content + * @return this commit builder + * @throws Exception + */ public CommitBuilder add(String path, String content) throws Exception { return add(path, blob(content)); } + /** + * Add file with given path and blob + * + * @param path + * path of the file + * @param id + * blob for this file + * @return this commit builder + * @throws Exception + */ public CommitBuilder add(String path, RevBlob id) throws Exception { return edit(new PathEdit(path) { @@ -1105,6 +1155,13 @@ public void apply(DirCacheEntry ent) { }); } + /** + * Edit the index + * + * @param edit + * the index record update + * @return this commit builder + */ public CommitBuilder edit(PathEdit edit) { DirCacheEditor e = tree.editor(); e.add(edit); @@ -1112,6 +1169,13 @@ public CommitBuilder edit(PathEdit edit) { return this; } + /** + * Remove a file + * + * @param path + * path of the file + * @return this commit builder + */ public CommitBuilder rm(String path) { DirCacheEditor e = tree.editor(); e.add(new DeletePath(path)); @@ -1120,49 +1184,111 @@ public CommitBuilder rm(String path) { return this; } + /** + * Set commit message + * + * @param m + * the message + * @return this commit builder + */ public CommitBuilder message(String m) { message = m; return this; } + /** + * Get the commit message + * + * @return the commit message + */ public String message() { return message; } + /** + * Tick the clock + * + * @param secs + * number of seconds + * @return this commit builder + */ public CommitBuilder tick(int secs) { tick = secs; return this; } + /** + * Set author and committer identity + * + * @param ident + * identity to set + * @return this commit builder + */ public CommitBuilder ident(PersonIdent ident) { author = ident; committer = ident; return this; } + /** + * Set the author identity + * + * @param a + * the author's identity + * @return this commit builder + */ public CommitBuilder author(PersonIdent a) { author = a; return this; } + /** + * Get the author identity + * + * @return the author identity + */ public PersonIdent author() { return author; } + /** + * Set the committer identity + * + * @param c + * the committer identity + * @return this commit builder + */ public CommitBuilder committer(PersonIdent c) { committer = c; return this; } + /** + * Get the committer identity + * + * @return the committer identity + */ public PersonIdent committer() { return committer; } + /** + * Insert changeId + * + * @return this commit builder + */ public CommitBuilder insertChangeId() { changeId = ""; return this; } + /** + * Insert given changeId + * + * @param c + * changeId + * @return this commit builder + */ public CommitBuilder insertChangeId(String c) { // Validate, but store as a string so we can use "" as a sentinel. ObjectId.fromString(c); @@ -1170,6 +1296,13 @@ public CommitBuilder insertChangeId(String c) { return this; } + /** + * Create the commit + * + * @return the new commit + * @throws Exception + * if creation failed + */ public RevCommit create() throws Exception { if (self == null) { TestRepository.this.tick(tick); @@ -1230,6 +1363,12 @@ private void insertChangeId(org.eclipse.jgit.lib.CommitBuilder c) { + cid.getName() + "\n"); //$NON-NLS-1$ } + /** + * Create child commit builder + * + * @return child commit builder + * @throws Exception + */ public CommitBuilder child() throws Exception { return new CommitBuilder(this); } From 773cbb5f6373dec45a1520666ab6dc5727d34808 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 18 Jul 2019 13:26:22 +0200 Subject: [PATCH 36/47] Fix org.eclipse.jdt.core.prefs of org.eclipse.jgit.junit Use the same JDT core settings as used in org.eclipse.jgit but ignore non-externalized strings. Change-Id: If30013c76a197e571601a8abc882ac6a99592374 Signed-off-by: Matthias Sohn --- .../.settings/org.eclipse.jdt.core.prefs | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/org.eclipse.jgit.junit/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.junit/.settings/org.eclipse.jdt.core.prefs index 794592dee..6251b7693 100644 --- a/org.eclipse.jgit.junit/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.junit/.settings/org.eclipse.jdt.core.prefs @@ -1,10 +1,13 @@ eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=enabled org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore -org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull -org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault -org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable -org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jgit.annotations.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jgit.annotations.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jgit.annotations.Nullable +org.eclipse.jdt.core.compiler.annotation.nullable.secondary= +org.eclipse.jdt.core.compiler.annotation.nullanalysis=enabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 @@ -14,6 +17,7 @@ org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.doc.comment.support=enabled +org.eclipse.jdt.core.compiler.problem.APILeak=warning org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.autoboxing=warning @@ -48,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error -org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore +org.eclipse.jdt.core.compiler.problem.missingJavadocComments=error org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag @@ -64,14 +68,16 @@ org.eclipse.jdt.core.compiler.problem.noEffectAssignment=error org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=error org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error org.eclipse.jdt.core.compiler.problem.nullReference=error org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error -org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=ignore org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=error -org.eclipse.jdt.core.compiler.problem.potentialNullReference=warning +org.eclipse.jdt.core.compiler.problem.potentialNullReference=error org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore org.eclipse.jdt.core.compiler.problem.rawTypeReference=ignore org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning @@ -86,16 +92,21 @@ org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.terminalDeprecation=warning org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=warning org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentType=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentTypeStrict=disabled +org.eclipse.jdt.core.compiler.problem.unlikelyEqualsArgumentType=info org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=error org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=error +org.eclipse.jdt.core.compiler.problem.unstableAutoModuleName=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=warning org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled From de6df3bdd92fe61cb6d4b648aec07593d2c304f8 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 12 Jul 2019 09:46:25 +0200 Subject: [PATCH 37/47] Repeat RefDirectoryTest.testGetRef_DiscoversModifiedLoose 100 times This should help to detect if measured fsTimeResolution is too small. Change-Id: Id1f54dbdedb52b17859904e47776fa3a5887b8be Signed-off-by: Matthias Sohn --- .../eclipse/jgit/internal/storage/file/RefDirectoryTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java index ac0a34ded..24e3bc077 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java @@ -71,6 +71,7 @@ import org.eclipse.jgit.events.RefsChangedEvent; import org.eclipse.jgit.events.RefsChangedListener; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; +import org.eclipse.jgit.junit.Repeat; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Ref; @@ -644,6 +645,7 @@ public void testGetRefs_DiscoversModifiedLoose() throws IOException { assertEquals(B, all.get(HEAD).getObjectId()); } + @Repeat(n = 100, abortOnFailure = false) @Test public void testGetRef_DiscoversModifiedLoose() throws IOException { Map all; From 130aa312620cb016dbe8d7ad71bcf38512eb85fe Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 12 Jul 2019 09:49:13 +0200 Subject: [PATCH 38/47] Add test for racy git detection in FileSnapshot Repeat the test 10000 times to get statistics if measured fsTimestampResolution is working in practice to detect racy git situations. Add a class to compute statistics for this test. Log delta between lastModified and time when FileSnapshot failed to detect modification. This happens if the racy git limit determined by measuring filesystem timestamp resolution and clock resolution is too small. If it would be correct FileSnapshot would always detect modification or mark it modified if time since modification is smaller than the racy git limit. Change-Id: Iabe7af1a7211ca58480f8902d4fa4e366932fc77 Signed-off-by: Matthias Sohn --- .../storage/file/FileSnapshotTest.java | 52 +++++++ .../tst/org/eclipse/jgit/util/StatsTest.java | 137 ++++++++++++++++++ org.eclipse.jgit/.settings/.api_filters | 8 + .../internal/storage/file/FileSnapshot.java | 23 ++- .../src/org/eclipse/jgit/util/Stats.java | 132 +++++++++++++++++ 5 files changed, 350 insertions(+), 2 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 1dea47084..9eb55db09 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -42,9 +42,13 @@ */ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.junit.JGitTestUtil.read; +import static org.eclipse.jgit.junit.JGitTestUtil.write; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -54,17 +58,23 @@ import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; import org.junit.After; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileSnapshotTest { + private static final Logger LOG = LoggerFactory + .getLogger(FileSnapshotTest.class); private Path trash; @@ -205,6 +215,48 @@ public void fileSnapshotEquals() throws Exception { assertTrue(fs2.equals(fs1)); } + @SuppressWarnings("boxing") + @Test + public void detectFileModified() throws IOException { + int failures = 0; + long racyNanos = 0; + final int COUNT = 10000; + ArrayList deltas = new ArrayList<>(); + File f = createFile("test").toFile(); + for (int i = 0; i < COUNT; i++) { + write(f, "a"); + FileSnapshot snapshot = FileSnapshot.save(f); + assertEquals("file should contain 'a'", "a", read(f)); + write(f, "b"); + if (!snapshot.isModified(f)) { + deltas.add(snapshot.lastDelta()); + racyNanos = snapshot.lastRacyNanos(); + failures++; + } + assertEquals("file should contain 'b'", "b", read(f)); + } + if (failures > 0) { + Stats stats = new Stats(); + LOG.debug( + "delta [ns] since modification FileSnapshot failed to detect"); + for (Long d : deltas) { + stats.add(d); + LOG.debug(String.format("%,d", d)); + } + LOG.error( + "count, failures, racy limit [ns], delta min [ns]," + + " delta max [ns], delta avg [ns]," + + " delta stddev [ns]"); + LOG.error(String.format( + "%,d, %,d, %,d, %,.0f, %,.0f, %,.0f, %,.0f", COUNT, + failures, racyNanos, stats.min(), stats.max(), + stats.avg(), stats.stddev())); + } + assertTrue( + "FileSnapshot: number of failures to detect file modifications should be 0", + failures == 0); + } + private Path createFile(String string) throws IOException { Files.createDirectories(trash); return Files.createTempFile(trash, string, "tdat"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java new file mode 100644 index 000000000..8b253828c --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StatsTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2019, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.eclipse.jgit.util.Stats; +import org.junit.Test; + +public class StatsTest { + @Test + public void testStatsTrivial() { + Stats s = new Stats(); + s.add(1); + s.add(1); + s.add(1); + assertEquals(3, s.count()); + assertEquals(1.0, s.min(), 1E-6); + assertEquals(1.0, s.max(), 1E-6); + assertEquals(1.0, s.avg(), 1E-6); + assertEquals(0.0, s.var(), 1E-6); + assertEquals(0.0, s.stddev(), 1E-6); + } + + @Test + public void testStats() { + Stats s = new Stats(); + s.add(1); + s.add(2); + s.add(3); + s.add(4); + assertEquals(4, s.count()); + assertEquals(1.0, s.min(), 1E-6); + assertEquals(4.0, s.max(), 1E-6); + assertEquals(2.5, s.avg(), 1E-6); + assertEquals(1.666667, s.var(), 1E-6); + assertEquals(1.290994, s.stddev(), 1E-6); + } + + @Test + /** + * see + * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Example + */ + public void testStatsCancellationExample1() { + Stats s = new Stats(); + s.add(1E8 + 4); + s.add(1E8 + 7); + s.add(1E8 + 13); + s.add(1E8 + 16); + assertEquals(4, s.count()); + assertEquals(1E8 + 4, s.min(), 1E-6); + assertEquals(1E8 + 16, s.max(), 1E-6); + assertEquals(1E8 + 10, s.avg(), 1E-6); + assertEquals(30, s.var(), 1E-6); + assertEquals(5.477226, s.stddev(), 1E-6); + } + + @Test + /** + * see + * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Example + */ + public void testStatsCancellationExample2() { + Stats s = new Stats(); + s.add(1E9 + 4); + s.add(1E9 + 7); + s.add(1E9 + 13); + s.add(1E9 + 16); + assertEquals(4, s.count()); + assertEquals(1E9 + 4, s.min(), 1E-6); + assertEquals(1E9 + 16, s.max(), 1E-6); + assertEquals(1E9 + 10, s.avg(), 1E-6); + assertEquals(30, s.var(), 1E-6); + assertEquals(5.477226, s.stddev(), 1E-6); + } + + @Test + public void testNoValues() { + Stats s = new Stats(); + assertTrue(Double.isNaN(s.var())); + assertTrue(Double.isNaN(s.stddev())); + assertTrue(Double.isNaN(s.avg())); + assertTrue(Double.isNaN(s.min())); + assertTrue(Double.isNaN(s.max())); + s.add(42.3); + assertTrue(Double.isNaN(s.var())); + assertTrue(Double.isNaN(s.stddev())); + assertEquals(42.3, s.avg(), 1E-6); + assertEquals(42.3, s.max(), 1E-6); + assertEquals(42.3, s.min(), 1E-6); + s.add(42.3); + assertEquals(0, s.var(), 1E-6); + assertEquals(0, s.stddev(), 1E-6); + } +} diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 01215a8c7..59bafc52e 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -226,4 +226,12 @@ + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index ec71783c6..2a490a4a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -283,6 +283,10 @@ protected FileSnapshot(File file, boolean useConfig) { private boolean wasRacyClean; + private long delta; + + private long racyNanos; + private FileSnapshot(Instant read, Instant modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { this.file = null; @@ -466,6 +470,21 @@ boolean wasLastModifiedRacilyClean() { return wasRacyClean; } + /** + * @return the delta in nanoseconds between lastModified and lastRead during + * last racy check + */ + long lastDelta() { + return delta; + } + + /** + * @return the racyNanos threshold in nanoseconds during last racy check + */ + long lastRacyNanos() { + return racyNanos; + } + /** {@inheritDoc} */ @SuppressWarnings("nls") @Override @@ -483,8 +502,8 @@ public String toString() { private boolean isRacyClean(Instant read) { // add a 10% safety margin - long racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; - long delta = Duration.between(lastModified, read).toNanos(); + racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; + delta = Duration.between(lastModified, read).toNanos(); wasRacyClean = delta <= racyNanos; if (LOG.isDebugEnabled()) { LOG.debug( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java new file mode 100644 index 000000000..e9307d342 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2019, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +/** + * Simple double statistics, computed incrementally, variance and standard + * deviation using Welford's online algorithm, see + * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + * + * @since 5.1.9 + */ +public class Stats { + private int n = 0; + + private double avg = 0.0; + + private double min = 0.0; + + private double max = 0.0; + + private double sum = 0.0; + + /** + * Add a value + * + * @param x + * value + */ + public void add(double x) { + n++; + min = n == 1 ? x : Math.min(min, x); + max = n == 1 ? x : Math.max(max, x); + double d = x - avg; + avg += d / n; + sum += d * d * (n - 1) / n; + } + + /** + * @return number of the added values + */ + public int count() { + return n; + } + + /** + * @return minimum of the added values + */ + public double min() { + if (n < 1) { + return Double.NaN; + } + return min; + } + + /** + * @return maximum of the added values + */ + public double max() { + if (n < 1) { + return Double.NaN; + } + return max; + } + + /** + * @return average of the added values + */ + + public double avg() { + if (n < 1) { + return Double.NaN; + } + return avg; + } + + /** + * @return variance of the added values + */ + public double var() { + if (n < 2) { + return Double.NaN; + } + return sum / (n - 1); + } + + /** + * @return standard deviation of the added values + */ + public double stddev() { + return Math.sqrt(this.var()); + } +} From d17efe880d0bb16c966dcc6fb55a98f46afc4508 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 26 Jul 2019 15:53:29 +0200 Subject: [PATCH 39/47] Fix FileAttributeCache.toString() We should not list the complete cache but only show the cache entry at hand. Change-Id: I22be2a4dcbf0145155e23f2389bfcf5662cf23a6 Signed-off-by: Matthias Sohn --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index e33c4bff9..081776f08 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -86,7 +86,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; @@ -438,12 +437,10 @@ private FileStoreAttributeCache( @SuppressWarnings("nls") @Override public String toString() { - return "FileStoreAttributeCache[" + attributeCache.keySet() - .stream() - .map(key -> "FileStore[" + key + "]: fsTimestampResolution=" - + attributeCache.get(key).getFsTimestampResolution()) - .collect(Collectors.joining(",\n")) + "]"; + return "FileStoreAttributeCache [fsTimestampResolution=" + + fsTimestampResolution + "]"; } + } /** The auto-detected implementation selected for this operating system and JRE. */ From 902935c38c0f1e9e8ecb0fce24c63cb06c07c963 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 16 Jul 2019 16:34:56 +0200 Subject: [PATCH 40/47] Reuse FileUtils to recursively delete files created by tests Replace redundant complex implementation of recursive delete by the one in FileUtils. Change-Id: Iced1468b96c4f32381a9cf0c651b2bf6a9a9af35 Signed-off-by: Matthias Sohn --- .../junit/LocalDiskRepositoryTestCase.java | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index af688d234..62dfc5d9c 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -51,6 +51,7 @@ import java.io.File; import java.io.IOException; +import java.io.PrintStream; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -237,35 +238,30 @@ protected void recursiveDelete(File dir) { private static boolean recursiveDelete(final File dir, boolean silent, boolean failOnError) { assert !(silent && failOnError); - if (!dir.exists()) - return silent; - final File[] ls = dir.listFiles(); - if (ls != null) - for (int k = 0; k < ls.length; k++) { - final File e = ls[k]; - if (e.isDirectory()) - silent = recursiveDelete(e, silent, failOnError); - else if (!e.delete()) { - if (!silent) - reportDeleteFailure(failOnError, e); - silent = !failOnError; - } - } - if (!dir.delete()) { - if (!silent) - reportDeleteFailure(failOnError, dir); - silent = !failOnError; + int options = FileUtils.RECURSIVE | FileUtils.RETRY + | FileUtils.SKIP_MISSING; + if (silent) { + options |= FileUtils.IGNORE_ERRORS; } - return silent; + try { + FileUtils.delete(dir, options); + } catch (IOException e) { + reportDeleteFailure(failOnError, dir, e); + return !failOnError; + } + return true; } - private static void reportDeleteFailure(boolean failOnError, File e) { + private static void reportDeleteFailure(boolean failOnError, File f, + Exception cause) { String severity = failOnError ? "ERROR" : "WARNING"; - String msg = severity + ": Failed to delete " + e; - if (failOnError) + String msg = severity + ": Failed to delete " + f; + if (failOnError) { fail(msg); - else + } else { System.err.println(msg); + } + cause.printStackTrace(new PrintStream(System.err)); } /** Constant MOD_TIME=1 */ From 5911521ba6d47868871c4b5f240c71af827b7aa2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 15 Jul 2019 15:00:09 +0200 Subject: [PATCH 41/47] Measure minimum racy interval to auto-configure FileSnapshot By running FileSnapshotTest#detectFileModified we found that the sum of measured filesystem timestamp resolution and measured clock resolution may yield a too small interval after a file has been modified which we need to consider racily clean. In our tests we didn't find this behavior on all systems we tested on, e.g. on MacOS using APFS and Java 8 and 11 this effect was not observed. On Linux (SLES 15, kernel 4.12.14-150.22-default) we collected the following test results using Java 8 and 11: In 23-98% of 10000 test runs (depending on filesystem type and Java version) the test failed, which means the effective interval which needs to be considered racily clean after a file was modified is larger than the measured file timestamp resolution. "delta" is the observed interval after a file has been modified but FileSnapshot did not yet detect the modification: "resolution" is the measured sum of file timestamp resolution and clock resolution seen in Java. Java version filesystem failures resolution min delta max delta 1.8.0_212-b04 btrfs 98.6% 1 ms 3.6 ms 6.6 ms 1.8.0_212-b04 ext4 82.6% 3 ms 1.1 ms 4.1 ms 1.8.0_212-b04 xfs 23.8% 4 ms 3.7 ms 3.9 ms 1.8.0_212-b04 zfs 23.1% 3 ms 4.8 ms 5.0 ms 11.0.3+7 btrfs 98.1% 3 us 0.7 ms 4.7 ms 11.0.3+7 ext4 98.1% 6 us 0.7 ms 4.7 ms 11.0.3+7 xfs 98.5% 7 us 0.1 ms 8.0 ms 11.0.3+7 zfs 98.4% 7 us 0.7 ms 5.2 ms Mac OS 1.8.0_212 APFS 0% 1 s 11.0.3+7 APFS 0% 6 us The observed delta is not distributed according to a normal gaussian distribution but rather random in the observed range between "min delta" and "max delta". Run this test after measuring file timestamp resolution in FS.FileAttributeCache to auto-configure JGit since it's unclear what mechanism is causing this effect. In FileSnapshot#isRacyClean use the maximum of the measured timestamp resolution and the measured "delta" as explained above to decide if a given FileSnapshot is to be considered racily clean. Add a 30% safety margin to ensure we are on the safe side. Change-Id: I1c8bb59f6486f174b7bbdc63072777ddbe06694d Signed-off-by: Matthias Sohn --- .../jgit/ant/tasks/GitCloneTaskTest.java | 2 +- .../junit/LocalDiskRepositoryTestCase.java | 2 +- .../jgit/junit/RepositoryTestCase.java | 3 +- .../jgit/lfs/server/fs/LfsServerTest.java | 2 +- .../tst-rsrc/log4j.properties | 5 +- .../storage/file/FileSnapshotTest.java | 28 ++- .../storage/file/FileBasedConfigTest.java | 2 +- .../tst/org/eclipse/jgit/util/FSTest.java | 3 +- org.eclipse.jgit/.settings/.api_filters | 28 +-- .../internal/storage/file/FileSnapshot.java | 48 +++-- .../src/org/eclipse/jgit/lib/Constants.java | 11 - .../src/org/eclipse/jgit/util/FS.java | 203 +++++++++++++++--- 12 files changed, 248 insertions(+), 89 deletions(-) diff --git a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java index 1d7187a31..9f9d459a6 100644 --- a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java +++ b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java @@ -66,7 +66,7 @@ public class GitCloneTaskTest extends LocalDiskRepositoryTestCase { @Before public void before() throws IOException { dest = createTempFile(); - FS.getFsTimerResolution(dest.toPath().getParent()); + FS.getFileStoreAttributeCache(dest.toPath().getParent()); project = new Project(); project.init(); enableLogging(); diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index 62dfc5d9c..fb8295fa4 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -130,7 +130,7 @@ public void setUp() throws Exception { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(tmp.toPath().getParent()); + FS.getFileStoreAttributeCache(tmp.toPath().getParent()); mockSystemReader = new MockSystemReader(); mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index 49f5c5feb..ebd13e411 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -378,7 +378,8 @@ public static Instant fsTick(File lastFile) tmp = File.createTempFile("fsTickTmpFile", null, lastFile.getParentFile()); } - long res = FS.getFsTimerResolution(tmp.toPath()).toNanos(); + long res = FS.getFileStoreAttributeCache(tmp.toPath()) + .getFsTimestampResolution().toNanos(); long sleepTime = res / 10; try { Instant startTime = fs.lastModifiedInstant(lastFile); diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 63af6eb52..92a6ec351 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -123,7 +123,7 @@ public void setup() throws Exception { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFsTimerResolution(tmp.getParent()); + FS.getFileStoreAttributeCache(tmp.getParent()); server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); diff --git a/org.eclipse.jgit.test/tst-rsrc/log4j.properties b/org.eclipse.jgit.test/tst-rsrc/log4j.properties index a48a4022f..ee1ac3515 100644 --- a/org.eclipse.jgit.test/tst-rsrc/log4j.properties +++ b/org.eclipse.jgit.test/tst-rsrc/log4j.properties @@ -8,4 +8,7 @@ log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n #log4j.appender.fileLogger.bufferedIO = true -#log4j.appender.fileLogger.bufferSize = 1024 \ No newline at end of file +#log4j.appender.fileLogger.bufferSize = 4096 + +#log4j.logger.org.eclipse.jgit.util.FS = DEBUG +#log4j.logger.org.eclipse.jgit.internal.storage.file.FileSnapshot = DEBUG diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 9eb55db09..012407f71 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.FileStoreAttributeCache; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; @@ -78,14 +79,15 @@ public class FileSnapshotTest { private Path trash; - private Duration fsTimerResolution; + private FileStoreAttributeCache fsAttrCache; @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - fsTimerResolution = FS.getFsTimerResolution(trash.getParent()); + fsAttrCache = FS + .getFileStoreAttributeCache(trash.getParent()); } @Before @@ -131,11 +133,13 @@ public void testNewFileWithWait() throws Exception { // if filesystem timestamp resolution is high the snapshot won't be // racily clean Assume.assumeTrue( - fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); + fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); waitNextTick(f1); FileSnapshot save = FileSnapshot.save(f1.toFile()); - TimeUnit.NANOSECONDS.sleep(fsTimerResolution.dividedBy(2).toNanos()); + TimeUnit.NANOSECONDS.sleep( + fsAttrCache.getFsTimestampResolution().dividedBy(2).toNanos()); assertTrue(save.isModified(f1.toFile())); } @@ -149,7 +153,8 @@ public void testNewFileNoWait() throws Exception { // if filesystem timestamp resolution is high the snapshot won't be // racily clean Assume.assumeTrue( - fsTimerResolution.compareTo(Duration.ofMillis(10)) > 0); + fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); Path f1 = createFile("newfile"); FileSnapshot save = FileSnapshot.save(f1.toFile()); assertTrue(save.isModified(f1.toFile())); @@ -230,7 +235,7 @@ public void detectFileModified() throws IOException { write(f, "b"); if (!snapshot.isModified(f)) { deltas.add(snapshot.lastDelta()); - racyNanos = snapshot.lastRacyNanos(); + racyNanos = snapshot.lastRacyThreshold(); failures++; } assertEquals("file should contain 'b'", "b", read(f)); @@ -244,7 +249,7 @@ public void detectFileModified() throws IOException { LOG.debug(String.format("%,d", d)); } LOG.error( - "count, failures, racy limit [ns], delta min [ns]," + "count, failures, eff. racy threshold [ns], delta min [ns]," + " delta max [ns], delta avg [ns]," + " delta stddev [ns]"); LOG.error(String.format( @@ -253,7 +258,14 @@ public void detectFileModified() throws IOException { stats.avg(), stats.stddev())); } assertTrue( - "FileSnapshot: number of failures to detect file modifications should be 0", + String.format( + "FileSnapshot: failures to detect file modifications" + + " %d out of %d\n" + + "timestamp resolution %d µs" + + " min racy threshold %d µs" + , failures, COUNT, + fsAttrCache.getFsTimestampResolution().toNanos() / 1000, + fsAttrCache.getMinimalRacyInterval().toNanos() / 1000), failures == 0); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index d3686285e..77f5febc1 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -83,7 +83,7 @@ public class FileBasedConfigTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); - FS.getFsTimerResolution(trash.getParent()); + FS.getFileStoreAttributeCache(trash.getParent()); } @After diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index bde8a8a6b..63e295ec8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -203,7 +203,8 @@ public void testFsTimestampResolution() throws Exception { .ofPattern("uuuu-MMM-dd HH:mm:ss.nnnnnnnnn", Locale.ENGLISH) .withZone(ZoneId.systemDefault()); Path dir = Files.createTempDirectory("probe-filesystem"); - Duration resolution = FS.getFsTimerResolution(dir); + Duration resolution = FS.getFileStoreAttributeCache(dir) + .getFsTimestampResolution(); long resolutionNs = resolution.toNanos(); assertTrue(resolutionNs > 0); for (int i = 0; i < 10; i++) { diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 59bafc52e..a027caaf0 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -56,14 +56,6 @@ - - - - - - - - @@ -179,6 +171,12 @@ + + + + + + @@ -203,12 +201,6 @@ - - - - - - @@ -218,6 +210,14 @@ + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 2a490a4a1..aa9f1cc45 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; +import static org.eclipse.jgit.util.FS.FileStoreAttributeCache.FALLBACK_FILESTORE_ATTRIBUTES; import java.io.File; import java.io.IOException; @@ -58,6 +58,7 @@ import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.FileStoreAttributeCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,8 +214,8 @@ public static FileSnapshot save(Instant modified) { * When set to {@link #UNKNOWN_SIZE} the size is not considered for modification checks. */ private final long size; - /** measured filesystem timestamp resolution */ - private Duration fsTimestampResolution; + /** measured FileStore attributes */ + private FileStoreAttributeCache fileStoreAttributeCache; /** * Object that uniquely identifies the given file, or {@code @@ -252,9 +253,9 @@ protected FileSnapshot(File file) { protected FileSnapshot(File file, boolean useConfig) { this.file = file; this.lastRead = Instant.now(); - this.fsTimestampResolution = useConfig - ? FS.getFsTimerResolution(file.toPath().getParent()) - : FALLBACK_TIMESTAMP_RESOLUTION; + this.fileStoreAttributeCache = useConfig + ? FS.getFileStoreAttributeCache(file.toPath().getParent()) + : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(file); @@ -285,14 +286,15 @@ protected FileSnapshot(File file, boolean useConfig) { private long delta; - private long racyNanos; + private long racyThreshold; private FileSnapshot(Instant read, Instant modified, long size, @NonNull Duration fsTimestampResolution, @NonNull Object fileKey) { this.file = null; this.lastRead = read; this.lastModified = modified; - this.fsTimestampResolution = fsTimestampResolution; + this.fileStoreAttributeCache = new FileStoreAttributeCache( + fsTimestampResolution); this.size = size; this.fileKey = fileKey; } @@ -397,9 +399,10 @@ public void setClean(FileSnapshot other) { * if sleep was interrupted */ public void waitUntilNotRacy() throws InterruptedException { + long timestampResolution = fileStoreAttributeCache + .getFsTimestampResolution().toNanos(); while (isRacyClean(Instant.now())) { - TimeUnit.NANOSECONDS - .sleep((fsTimestampResolution.toNanos() + 1) * 11 / 10); + TimeUnit.NANOSECONDS.sleep(timestampResolution); } } @@ -474,15 +477,16 @@ boolean wasLastModifiedRacilyClean() { * @return the delta in nanoseconds between lastModified and lastRead during * last racy check */ - long lastDelta() { + public long lastDelta() { return delta; } /** - * @return the racyNanos threshold in nanoseconds during last racy check + * @return the racyLimitNanos threshold in nanoseconds during last racy + * check */ - long lastRacyNanos() { - return racyNanos; + public long lastRacyThreshold() { + return racyThreshold; } /** {@inheritDoc} */ @@ -501,20 +505,28 @@ public String toString() { } private boolean isRacyClean(Instant read) { - // add a 10% safety margin - racyNanos = (fsTimestampResolution.toNanos() + 1) * 11 / 10; + racyThreshold = getEffectiveRacyThreshold(); delta = Duration.between(lastModified, read).toNanos(); - wasRacyClean = delta <= racyNanos; + wasRacyClean = delta <= racyThreshold; if (LOG.isDebugEnabled()) { LOG.debug( "file={}, isRacyClean={}, read={}, lastModified={}, delta={} ns, racy<={} ns", //$NON-NLS-1$ file, Boolean.valueOf(wasRacyClean), dateFmt.format(read), dateFmt.format(lastModified), Long.valueOf(delta), - Long.valueOf(racyNanos)); + Long.valueOf(racyThreshold)); } return wasRacyClean; } + private long getEffectiveRacyThreshold() { + long timestampResolution = fileStoreAttributeCache + .getFsTimestampResolution().toNanos(); + long minRacyInterval = fileStoreAttributeCache.getMinimalRacyInterval() + .toNanos(); + // add a 30% safety margin + return Math.max(timestampResolution, minRacyInterval) * 13 / 10; + } + private boolean isModified(Instant currLastModified) { // Any difference indicates the path was modified. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 94fc10038..4c5519696 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -52,7 +52,6 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; -import java.time.Duration; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.internal.JGitText; @@ -723,16 +722,6 @@ public static byte[] encode(String str) { */ public static final String LOCK_SUFFIX = ".lock"; //$NON-NLS-1$ - /** - * Fallback filesystem timestamp resolution used when we can't measure the - * resolution. The last modified time granularity of FAT filesystems is 2 - * seconds. - * - * @since 5.1.9 - */ - public static final Duration FALLBACK_TIMESTAMP_RESOLUTION = Duration - .ofMillis(2000); - private Constants() { // Hide the default constructor } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 081776f08..08dab3201 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -45,7 +45,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.Instant.EPOCH; -import static org.eclipse.jgit.lib.Constants.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -55,7 +54,9 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.PrintStream; +import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; import java.nio.file.FileStore; @@ -68,6 +69,7 @@ import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -94,6 +96,7 @@ import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.FileSnapshot; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; @@ -200,11 +203,24 @@ public int getRc() { } } - private static final class FileStoreAttributeCache { + /** + * Attributes of FileStores on this system + * + * @since 5.1.9 + */ + public final static class FileStoreAttributeCache { private static final Duration UNDEFINED_RESOLUTION = Duration .ofNanos(Long.MAX_VALUE); + /** + * Fallback FileStore attributes used when we can't measure the + * filesystem timestamp resolution. The last modified time granularity + * of FAT filesystems is 2 seconds. + */ + public static final FileStoreAttributeCache FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributeCache( + Duration.ofMillis(2000)); + private static final Map attributeCache = new ConcurrentHashMap<>(); private static AtomicBoolean background = new AtomicBoolean(); @@ -216,36 +232,58 @@ private static void setBackground(boolean async) { } private static final String javaVersionPrefix = System - .getProperty("java.vm.vendor") + '|' //$NON-NLS-1$ - + System.getProperty("java.vm.version") + '|'; //$NON-NLS-1$ + .getProperty("java.vendor") + '|' //$NON-NLS-1$ + + System.getProperty("java.version") + '|'; //$NON-NLS-1$ - private static Duration getFsTimestampResolution(Path file) { - file = file.toAbsolutePath(); - Path dir = Files.isDirectory(file) ? file : file.getParent(); + private static final Duration FALLBACK_MIN_RACY_INTERVAL = Duration + .ofMillis(10); + + /** + * @param path + * file residing in the FileStore to get attributes for + * @return FileStoreAttributeCache entry for the given path. + */ + public static FileStoreAttributeCache get(Path path) { + path = path.toAbsolutePath(); + Path dir = Files.isDirectory(path) ? path : path.getParent(); + return getFileAttributeCache(dir); + } + + private static FileStoreAttributeCache getFileAttributeCache(Path dir) { FileStore s; try { if (Files.exists(dir)) { s = Files.getFileStore(dir); FileStoreAttributeCache c = attributeCache.get(s); if (c != null) { - return c.getFsTimestampResolution(); + return c; } if (!Files.isWritable(dir)) { // cannot measure resolution in a read-only directory - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug( + "{}: cannot measure timestamp resolution in read-only directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; } } else { // cannot determine FileStore of an unborn directory - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug( + "{}: cannot measure timestamp resolution of unborn directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; } - CompletableFuture> f = CompletableFuture + CompletableFuture> f = CompletableFuture .supplyAsync(() -> { Lock lock = locks.computeIfAbsent(s, l -> new ReentrantLock()); if (!lock.tryLock()) { + LOG.debug( + "{}: couldn't get lock to measure timestamp resolution in {}", //$NON-NLS-1$ + Thread.currentThread(), dir); return Optional.empty(); } - Optional resolution; + Optional cache = Optional + .empty(); try { // Some earlier future might have set the value // and removed itself since we checked for the @@ -253,28 +291,36 @@ private static Duration getFsTimestampResolution(Path file) { FileStoreAttributeCache c = attributeCache .get(s); if (c != null) { - return Optional - .of(c.getFsTimestampResolution()); + return Optional.of(c); } - resolution = measureFsTimestampResolution(s, - dir); + Optional resolution = measureFsTimestampResolution( + s, dir); if (resolution.isPresent()) { - FileStoreAttributeCache cache = new FileStoreAttributeCache( + c = new FileStoreAttributeCache( resolution.get()); - attributeCache.put(s, cache); - if (LOG.isDebugEnabled()) { - LOG.debug(cache.toString()); + attributeCache.put(s, c); + // for high timestamp resolution measure + // minimal racy interval + if (c.fsTimestampResolution + .toNanos() < 100_000_000L) { + c.minimalRacyInterval = measureMinimalRacyInterval( + dir); } + if (LOG.isDebugEnabled()) { + LOG.debug(c.toString()); + } + cache = Optional.of(c); } } finally { lock.unlock(); locks.remove(s); } - return resolution; + return cache; }); // even if measuring in background wait a little - if the result // arrives, it's better than returning the large fallback - Optional d = f.get(background.get() ? 50 : 2000, + Optional d = f.get( + background.get() ? 100 : 5000, TimeUnit.MILLISECONDS); if (d.isPresent()) { return d.get(); @@ -286,11 +332,79 @@ private static Duration getFsTimestampResolution(Path file) { } catch (TimeoutException | SecurityException e) { // use fallback } - return FALLBACK_TIMESTAMP_RESOLUTION; + LOG.debug("{}: use fallback timestamp resolution for directory {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + return FALLBACK_FILESTORE_ATTRIBUTES; + } + + @SuppressWarnings("boxing") + private static Duration measureMinimalRacyInterval(Path dir) { + LOG.debug("{}: start measure minimal racy interval in {}", //$NON-NLS-1$ + Thread.currentThread(), dir); + int failures = 0; + long racyNanos = 0; + final int COUNT = 1000; + ArrayList deltas = new ArrayList<>(); + Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ + try { + Files.createFile(probe); + for (int i = 0; i < COUNT; i++) { + write(probe, "a"); //$NON-NLS-1$ + FileSnapshot snapshot = FileSnapshot.save(probe.toFile()); + read(probe); + write(probe, "b"); //$NON-NLS-1$ + if (!snapshot.isModified(probe.toFile())) { + deltas.add(Long.valueOf(snapshot.lastDelta())); + racyNanos = snapshot.lastRacyThreshold(); + failures++; + } + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + return FALLBACK_MIN_RACY_INTERVAL; + } finally { + deleteProbe(probe); + } + if (failures > 0) { + Stats stats = new Stats(); + for (Long d : deltas) { + stats.add(d); + } + LOG.debug( + "delta [ns] since modification FileSnapshot failed to detect\n" //$NON-NLS-1$ + + "count, failures, racy limit [ns], delta min [ns]," //$NON-NLS-1$ + + " delta max [ns], delta avg [ns]," //$NON-NLS-1$ + + " delta stddev [ns]\n" //$NON-NLS-1$ + + "{}, {}, {}, {}, {}, {}, {}", //$NON-NLS-1$ + COUNT, failures, racyNanos, stats.min(), stats.max(), + stats.avg(), stats.stddev()); + return Duration + .ofNanos(Double.valueOf(stats.max()).longValue()); + } + // since no failures occurred using the measured filesystem + // timestamp resolution there is no need for minimal racy interval + LOG.debug("{}: no failures when measuring minimal racy interval", //$NON-NLS-1$ + Thread.currentThread()); + return Duration.ZERO; + } + + private static void write(Path p, String body) throws IOException { + FileUtils.mkdirs(p.getParent().toFile(), true); + try (Writer w = new OutputStreamWriter(Files.newOutputStream(p), + UTF_8)) { + w.write(body); + } + } + + private static String read(Path p) throws IOException { + final byte[] body = IO.readFully(p.toFile()); + return new String(body, 0, body.length, UTF_8); } private static Optional measureFsTimestampResolution( FileStore s, Path dir) { + LOG.debug("{}: start measure timestamp resolution {} in {}", //$NON-NLS-1$ + Thread.currentThread(), s, dir); Duration configured = readFileTimeResolution(s); if (!UNDEFINED_RESOLUTION.equals(configured)) { return Optional.of(configured); @@ -310,6 +424,8 @@ private static Optional measureFsTimestampResolution( Duration clockResolution = measureClockResolution(); fsResolution = fsResolution.plus(clockResolution); saveFileTimeResolution(s, fsResolution); + LOG.debug("{}: end measure timestamp resolution {} in {}", //$NON-NLS-1$ + Thread.currentThread(), s, dir); return Optional.of(fsResolution); } catch (AccessDeniedException e) { LOG.warn(e.getLocalizedMessage(), e); // see bug 548648 @@ -424,21 +540,45 @@ private static void saveFileTimeResolution(FileStore s, private final @NonNull Duration fsTimestampResolution; + private Duration minimalRacyInterval; + + /** + * @return the measured minimal interval after a file has been modified + * in which we cannot rely on lastModified to detect + * modifications + */ + public Duration getMinimalRacyInterval() { + return minimalRacyInterval; + } + + /** + * @return the measured filesystem timestamp resolution + */ @NonNull - Duration getFsTimestampResolution() { + public Duration getFsTimestampResolution() { return fsTimestampResolution; } - private FileStoreAttributeCache( + /** + * Construct a FileStoreAttributeCache entry for the given filesystem + * timestamp resolution + * + * @param fsTimestampResolution + */ + public FileStoreAttributeCache( @NonNull Duration fsTimestampResolution) { this.fsTimestampResolution = fsTimestampResolution; + this.minimalRacyInterval = Duration.ZERO; } - @SuppressWarnings("nls") + @SuppressWarnings({ "nls", "boxing" }) @Override public String toString() { - return "FileStoreAttributeCache [fsTimestampResolution=" - + fsTimestampResolution + "]"; + return String.format( + "FileStoreAttributeCache[fsTimestampResolution=%,d µs, " + + "minimalRacyInterval=%,d µs]", + fsTimestampResolution.toNanos() / 1000, + minimalRacyInterval.toNanos() / 1000); } } @@ -507,10 +647,11 @@ public static FS detect(Boolean cygwinUsed) { * the directory under which the probe file will be created to * measure the timer resolution. * @return measured filesystem timestamp resolution - * @since 5.2.3 + * @since 5.1.9 */ - public static Duration getFsTimerResolution(@NonNull Path dir) { - return FileStoreAttributeCache.getFsTimestampResolution(dir); + public static FileStoreAttributeCache getFileStoreAttributeCache( + @NonNull Path dir) { + return FileStoreAttributeCache.get(dir); } private volatile Holder userHome; From d45219baacab711abf3c4112146ca0522d984be2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 17 Jul 2019 16:31:42 +0200 Subject: [PATCH 42/47] Persist minimal racy threshold and allow manual configuration To enable persisting the minimal racy threshold per FileStore add a new config option to the user global git configuration: - Config section is "filesystem" - Config subsection is concatenation of - Java vendor (system property "java.vendor") - Java version (system property "java.version") - FileStore's name, on Windows we use the attribute volume:vsn instead since the name is not necessarily unique. - separated by '|' e.g. "AdoptOpenJDK|1.8.0_212-b03|/dev/disk1s1" The same prefix is used as for filesystem timestamp resolution, so both values are stored in the same config section - The config key for minmal racy threshold is "minRacyThreshold" as a time value, supported time units are those supported by DefaultTypedConfigGetter#getTimeUnit - measure for 3 seconds to limit runtime which depends on hardware, OS and Java version being used If the minimal racy threshold is configured for a given FileStore the configured value is used instead of measuring it. When the minimal racy threshold was measured it is persisted in the user global git configuration. Rename FileStoreAttributeCache to FileStoreAttributes since this class is now declared public in order to enable exposing all attributes in one object. Example: [filesystem "AdoptOpenJDK|11.0.3|/dev/disk1s1"] timestampResolution = 7000 nanoseconds minRacyThreshold = 3440 microseconds Change-Id: I22195e488453aae8d011b0a8e3276fe3d99deaea Signed-off-by: Matthias Sohn Also-By: Marc Strapetz --- .../jgit/ant/tasks/GitCloneTaskTest.java | 2 +- .../junit/LocalDiskRepositoryTestCase.java | 2 +- .../jgit/junit/RepositoryTestCase.java | 2 +- .../jgit/lfs/server/fs/LfsServerTest.java | 2 +- .../storage/file/FileSnapshotTest.java | 6 +- .../storage/file/FileBasedConfigTest.java | 2 +- .../tst/org/eclipse/jgit/util/FSTest.java | 2 +- org.eclipse.jgit/.settings/.api_filters | 22 +- .../internal/storage/file/FileSnapshot.java | 10 +- .../org/eclipse/jgit/lib/ConfigConstants.java | 7 + .../jgit/storage/file/FileBasedConfig.java | 30 ++- .../src/org/eclipse/jgit/util/FS.java | 202 +++++++++++------- 12 files changed, 195 insertions(+), 94 deletions(-) diff --git a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java index 9f9d459a6..8043d2b18 100644 --- a/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java +++ b/org.eclipse.jgit.ant.test/src/org/eclipse/jgit/ant/tasks/GitCloneTaskTest.java @@ -66,7 +66,7 @@ public class GitCloneTaskTest extends LocalDiskRepositoryTestCase { @Before public void before() throws IOException { dest = createTempFile(); - FS.getFileStoreAttributeCache(dest.toPath().getParent()); + FS.getFileStoreAttributes(dest.toPath().getParent()); project = new Project(); project.init(); enableLogging(); diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java index fb8295fa4..af23ad1e3 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/LocalDiskRepositoryTestCase.java @@ -130,7 +130,7 @@ public void setUp() throws Exception { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFileStoreAttributeCache(tmp.toPath().getParent()); + FS.getFileStoreAttributes(tmp.toPath().getParent()); mockSystemReader = new MockSystemReader(); mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index ebd13e411..5aacbbade 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -378,7 +378,7 @@ public static Instant fsTick(File lastFile) tmp = File.createTempFile("fsTickTmpFile", null, lastFile.getParentFile()); } - long res = FS.getFileStoreAttributeCache(tmp.toPath()) + long res = FS.getFileStoreAttributes(tmp.toPath()) .getFsTimestampResolution().toNanos(); long sleepTime = res / 10; try { diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java index 92a6ec351..ec44da4ca 100644 --- a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java +++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java @@ -123,7 +123,7 @@ public void setup() throws Exception { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement - FS.getFileStoreAttributeCache(tmp.getParent()); + FS.getFileStoreAttributes(tmp.getParent()); server = new AppServer(); ServletContextHandler app = server.addContext("/lfs"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 012407f71..40af9e2a0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -62,7 +62,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.util.FS; -import org.eclipse.jgit.util.FS.FileStoreAttributeCache; +import org.eclipse.jgit.util.FS.FileStoreAttributes; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Stats; import org.eclipse.jgit.util.SystemReader; @@ -79,7 +79,7 @@ public class FileSnapshotTest { private Path trash; - private FileStoreAttributeCache fsAttrCache; + private FileStoreAttributes fsAttrCache; @Before public void setUp() throws Exception { @@ -87,7 +87,7 @@ public void setUp() throws Exception { // measure timer resolution before the test to avoid time critical tests // are affected by time needed for measurement fsAttrCache = FS - .getFileStoreAttributeCache(trash.getParent()); + .getFileStoreAttributes(trash.getParent()); } @Before diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index 77f5febc1..1adddb5ea 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -83,7 +83,7 @@ public class FileBasedConfigTest { @Before public void setUp() throws Exception { trash = Files.createTempDirectory("tmp_"); - FS.getFileStoreAttributeCache(trash.getParent()); + FS.getFileStoreAttributes(trash.getParent()); } @After diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java index 63e295ec8..2054e1efa 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FSTest.java @@ -203,7 +203,7 @@ public void testFsTimestampResolution() throws Exception { .ofPattern("uuuu-MMM-dd HH:mm:ss.nnnnnnnnn", Locale.ENGLISH) .withZone(ZoneId.systemDefault()); Path dir = Files.createTempDirectory("probe-filesystem"); - Duration resolution = FS.getFileStoreAttributeCache(dir) + Duration resolution = FS.getFileStoreAttributes(dir) .getFsTimestampResolution(); long resolutionNs = resolution.toNanos(); assertTrue(resolutionNs > 0); diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index a027caaf0..8277735e2 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -49,6 +49,12 @@ + + + + + + @@ -72,6 +78,14 @@ + + + + + + + + @@ -174,7 +188,7 @@ - + @@ -192,7 +206,7 @@ - + @@ -210,11 +224,11 @@ - + - + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index aa9f1cc45..e81a88451 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.util.FS.FileStoreAttributeCache.FALLBACK_FILESTORE_ATTRIBUTES; +import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_FILESTORE_ATTRIBUTES; import java.io.File; import java.io.IOException; @@ -58,7 +58,7 @@ import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.util.FS; -import org.eclipse.jgit.util.FS.FileStoreAttributeCache; +import org.eclipse.jgit.util.FS.FileStoreAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -215,7 +215,7 @@ public static FileSnapshot save(Instant modified) { private final long size; /** measured FileStore attributes */ - private FileStoreAttributeCache fileStoreAttributeCache; + private FileStoreAttributes fileStoreAttributeCache; /** * Object that uniquely identifies the given file, or {@code @@ -254,7 +254,7 @@ protected FileSnapshot(File file, boolean useConfig) { this.file = file; this.lastRead = Instant.now(); this.fileStoreAttributeCache = useConfig - ? FS.getFileStoreAttributeCache(file.toPath().getParent()) + ? FS.getFileStoreAttributes(file.toPath().getParent()) : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { @@ -293,7 +293,7 @@ private FileSnapshot(Instant read, Instant modified, long size, this.file = null; this.lastRead = read; this.lastModified = modified; - this.fileStoreAttributeCache = new FileStoreAttributeCache( + this.fileStoreAttributeCache = new FileStoreAttributes( fsTimestampResolution); this.size = size; this.fileKey = fileKey; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 4f636d455..82ccd7b03 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -444,4 +444,11 @@ public final class ConfigConstants { * @since 5.1.9 */ public static final String CONFIG_KEY_TIMESTAMP_RESOLUTION = "timestampResolution"; + + /** + * The "minRacyThreshold" key + * + * @since 5.1.9 + */ + public static final String CONFIG_KEY_MIN_RACY_THRESHOLD = "minRacyThreshold"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java index 3a41643e6..633632dc0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java @@ -149,13 +149,37 @@ public final File getFile() { */ @Override public void load() throws IOException, ConfigInvalidException { + load(true); + } + + /** + * Load the configuration as a Git text style configuration file. + *

+ * If the file does not exist, this configuration is cleared, and thus + * behaves the same as though the file exists, but is empty. + * + * @param useFileSnapshotWithConfig + * if {@code true} use the FileSnapshot with config, otherwise + * use it without config + * @throws IOException + * if IO failed + * @throws ConfigInvalidException + * if config is invalid + * @since 5.1.9 + */ + public void load(boolean useFileSnapshotWithConfig) + throws IOException, ConfigInvalidException { final int maxStaleRetries = 5; int retries = 0; while (true) { final FileSnapshot oldSnapshot = snapshot; - // don't use config in this snapshot to avoid endless recursion - final FileSnapshot newSnapshot = FileSnapshot - .saveNoConfig(getFile()); + final FileSnapshot newSnapshot; + if (useFileSnapshotWithConfig) { + newSnapshot = FileSnapshot.save(getFile()); + } else { + // don't use config in this snapshot to avoid endless recursion + newSnapshot = FileSnapshot.saveNoConfig(getFile()); + } try { final byte[] in = IO.readFully(getFile()); final ObjectId newHash = hash(in); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 08dab3201..16810e0dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -208,9 +208,9 @@ public int getRc() { * * @since 5.1.9 */ - public final static class FileStoreAttributeCache { + public final static class FileStoreAttributes { - private static final Duration UNDEFINED_RESOLUTION = Duration + private static final Duration UNDEFINED_DURATION = Duration .ofNanos(Long.MAX_VALUE); /** @@ -218,10 +218,10 @@ public final static class FileStoreAttributeCache { * filesystem timestamp resolution. The last modified time granularity * of FAT filesystems is 2 seconds. */ - public static final FileStoreAttributeCache FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributeCache( + public static final FileStoreAttributes FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributes( Duration.ofMillis(2000)); - private static final Map attributeCache = new ConcurrentHashMap<>(); + private static final Map attributeCache = new ConcurrentHashMap<>(); private static AtomicBoolean background = new AtomicBoolean(); @@ -239,22 +239,24 @@ private static void setBackground(boolean async) { .ofMillis(10); /** + * Get the FileStoreAttributes for the given FileStore + * * @param path * file residing in the FileStore to get attributes for - * @return FileStoreAttributeCache entry for the given path. + * @return FileStoreAttributes for the given path. */ - public static FileStoreAttributeCache get(Path path) { + public static FileStoreAttributes get(Path path) { path = path.toAbsolutePath(); Path dir = Files.isDirectory(path) ? path : path.getParent(); - return getFileAttributeCache(dir); + return getFileStoreAttributes(dir); } - private static FileStoreAttributeCache getFileAttributeCache(Path dir) { + private static FileStoreAttributes getFileStoreAttributes(Path dir) { FileStore s; try { if (Files.exists(dir)) { s = Files.getFileStore(dir); - FileStoreAttributeCache c = attributeCache.get(s); + FileStoreAttributes c = attributeCache.get(s); if (c != null) { return c; } @@ -272,7 +274,8 @@ private static FileStoreAttributeCache getFileAttributeCache(Path dir) { Thread.currentThread(), dir); return FALLBACK_FILESTORE_ATTRIBUTES; } - CompletableFuture> f = CompletableFuture + + CompletableFuture> f = CompletableFuture .supplyAsync(() -> { Lock lock = locks.computeIfAbsent(s, l -> new ReentrantLock()); @@ -282,21 +285,27 @@ private static FileStoreAttributeCache getFileAttributeCache(Path dir) { Thread.currentThread(), dir); return Optional.empty(); } - Optional cache = Optional + Optional attributes = Optional .empty(); try { // Some earlier future might have set the value // and removed itself since we checked for the // value above. Hence check cache again. - FileStoreAttributeCache c = attributeCache + FileStoreAttributes c = attributeCache .get(s); if (c != null) { return Optional.of(c); } + attributes = readFromConfig(s); + if (attributes.isPresent()) { + attributeCache.put(s, attributes.get()); + return attributes; + } + Optional resolution = measureFsTimestampResolution( s, dir); if (resolution.isPresent()) { - c = new FileStoreAttributeCache( + c = new FileStoreAttributes( resolution.get()); attributeCache.put(s, c); // for high timestamp resolution measure @@ -304,24 +313,28 @@ private static FileStoreAttributeCache getFileAttributeCache(Path dir) { if (c.fsTimestampResolution .toNanos() < 100_000_000L) { c.minimalRacyInterval = measureMinimalRacyInterval( - dir); + dir); } if (LOG.isDebugEnabled()) { LOG.debug(c.toString()); } - cache = Optional.of(c); + saveToConfig(s, c); } + attributes = Optional.of(c); } finally { lock.unlock(); locks.remove(s); } - return cache; + return attributes; }); + f.exceptionally(e -> { + LOG.error(e.getLocalizedMessage(), e); + return Optional.empty(); + }); // even if measuring in background wait a little - if the result // arrives, it's better than returning the large fallback - Optional d = f.get( - background.get() ? 100 : 5000, - TimeUnit.MILLISECONDS); + Optional d = background.get() ? f.get( + 100, TimeUnit.MILLISECONDS) : f.get(); if (d.isPresent()) { return d.get(); } @@ -341,14 +354,16 @@ private static FileStoreAttributeCache getFileAttributeCache(Path dir) { private static Duration measureMinimalRacyInterval(Path dir) { LOG.debug("{}: start measure minimal racy interval in {}", //$NON-NLS-1$ Thread.currentThread(), dir); + int n = 0; int failures = 0; long racyNanos = 0; - final int COUNT = 1000; ArrayList deltas = new ArrayList<>(); Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ + Instant end = Instant.now().plusSeconds(3); try { Files.createFile(probe); - for (int i = 0; i < COUNT; i++) { + do { + n++; write(probe, "a"); //$NON-NLS-1$ FileSnapshot snapshot = FileSnapshot.save(probe.toFile()); read(probe); @@ -358,7 +373,7 @@ private static Duration measureMinimalRacyInterval(Path dir) { racyNanos = snapshot.lastRacyThreshold(); failures++; } - } + } while (Instant.now().compareTo(end) < 0); } catch (IOException e) { LOG.error(e.getMessage(), e); return FALLBACK_MIN_RACY_INTERVAL; @@ -376,7 +391,7 @@ private static Duration measureMinimalRacyInterval(Path dir) { + " delta max [ns], delta avg [ns]," //$NON-NLS-1$ + " delta stddev [ns]\n" //$NON-NLS-1$ + "{}, {}, {}, {}, {}, {}, {}", //$NON-NLS-1$ - COUNT, failures, racyNanos, stats.min(), stats.max(), + n, failures, racyNanos, stats.min(), stats.max(), stats.avg(), stats.stddev()); return Duration .ofNanos(Double.valueOf(stats.max()).longValue()); @@ -405,10 +420,6 @@ private static Optional measureFsTimestampResolution( FileStore s, Path dir) { LOG.debug("{}: start measure timestamp resolution {} in {}", //$NON-NLS-1$ Thread.currentThread(), s, dir); - Duration configured = readFileTimeResolution(s); - if (!UNDEFINED_RESOLUTION.equals(configured)) { - return Optional.of(configured); - } Path probe = dir.resolve(".probe-" + UUID.randomUUID()); //$NON-NLS-1$ try { Files.createFile(probe); @@ -423,7 +434,6 @@ private static Optional measureFsTimestampResolution( Duration fsResolution = Duration.between(t1.toInstant(), t2.toInstant()); Duration clockResolution = measureClockResolution(); fsResolution = fsResolution.plus(clockResolution); - saveFileTimeResolution(s, fsResolution); LOG.debug("{}: end measure timestamp resolution {} in {}", //$NON-NLS-1$ Thread.currentThread(), s, dir); return Optional.of(fsResolution); @@ -454,20 +464,20 @@ private static Duration measureClockResolution() { } private static void deleteProbe(Path probe) { - if (Files.exists(probe)) { - try { - Files.delete(probe); - } catch (IOException e) { - LOG.error(e.getLocalizedMessage(), e); - } + try { + FileUtils.delete(probe.toFile(), + FileUtils.SKIP_MISSING | FileUtils.RETRY); + } catch (IOException e) { + LOG.error(e.getMessage(), e); } } - private static Duration readFileTimeResolution(FileStore s) { + private static Optional readFromConfig( + FileStore s) { FileBasedConfig userConfig = SystemReader.getInstance() .openUserConfig(null, FS.DETECTED); try { - userConfig.load(); + userConfig.load(false); } catch (IOException e) { LOG.error(MessageFormat.format(JGitText.get().readConfigFailed, userConfig.getFile().getAbsolutePath()), e); @@ -477,49 +487,65 @@ private static Duration readFileTimeResolution(FileStore s) { userConfig.getFile().getAbsolutePath(), e.getMessage())); } - Duration configured = Duration - .ofNanos(userConfig.getTimeUnit( - ConfigConstants.CONFIG_FILESYSTEM_SECTION, - javaVersionPrefix + s.name(), - ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, - UNDEFINED_RESOLUTION.toNanos(), - TimeUnit.NANOSECONDS)); - return configured; + String key = getConfigKey(s); + Duration resolution = Duration.ofNanos(userConfig.getTimeUnit( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, + ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, + UNDEFINED_DURATION.toNanos(), TimeUnit.NANOSECONDS)); + if (UNDEFINED_DURATION.equals(resolution)) { + return Optional.empty(); + } + Duration minRacyThreshold = Duration.ofNanos(userConfig.getTimeUnit( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, + ConfigConstants.CONFIG_KEY_MIN_RACY_THRESHOLD, + UNDEFINED_DURATION.toNanos(), TimeUnit.NANOSECONDS)); + FileStoreAttributes c = new FileStoreAttributes(resolution); + if (!UNDEFINED_DURATION.equals(minRacyThreshold)) { + c.minimalRacyInterval = minRacyThreshold; + } + return Optional.of(c); } - private static void saveFileTimeResolution(FileStore s, - Duration resolution) { + private static void saveToConfig(FileStore s, + FileStoreAttributes c) { FileBasedConfig userConfig = SystemReader.getInstance() .openUserConfig(null, FS.DETECTED); - long nanos = resolution.toNanos(); - TimeUnit unit; - if (nanos < 200_000L) { - unit = TimeUnit.NANOSECONDS; - } else if (nanos < 200_000_000L) { - unit = TimeUnit.MICROSECONDS; - } else { - unit = TimeUnit.MILLISECONDS; - } + long resolution = c.getFsTimestampResolution().toNanos(); + TimeUnit resolutionUnit = getUnit(resolution); + long resolutionValue = resolutionUnit.convert(resolution, + TimeUnit.NANOSECONDS); + + long minRacyThreshold = c.getMinimalRacyInterval().toNanos(); + TimeUnit minRacyThresholdUnit = getUnit(minRacyThreshold); + long minRacyThresholdValue = minRacyThresholdUnit + .convert(minRacyThreshold, TimeUnit.NANOSECONDS); final int max_retries = 5; int retries = 0; boolean succeeded = false; - long value = unit.convert(nanos, TimeUnit.NANOSECONDS); + String key = getConfigKey(s); while (!succeeded && retries < max_retries) { try { - userConfig.load(); + userConfig.load(false); userConfig.setString( - ConfigConstants.CONFIG_FILESYSTEM_SECTION, - javaVersionPrefix + s.name(), + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, ConfigConstants.CONFIG_KEY_TIMESTAMP_RESOLUTION, String.format("%d %s", //$NON-NLS-1$ - Long.valueOf(value), - unit.name().toLowerCase())); + Long.valueOf(resolutionValue), + resolutionUnit.name().toLowerCase())); + userConfig.setString( + ConfigConstants.CONFIG_FILESYSTEM_SECTION, key, + ConfigConstants.CONFIG_KEY_MIN_RACY_THRESHOLD, + String.format("%d %s", //$NON-NLS-1$ + Long.valueOf(minRacyThresholdValue), + minRacyThresholdUnit.name().toLowerCase())); userConfig.save(); succeeded = true; } catch (LockFailedException e) { // race with another thread, wait a bit and try again try { + LOG.warn(MessageFormat.format(JGitText.get().cannotLock, + userConfig.getFile().getAbsolutePath())); retries++; Thread.sleep(20); } catch (InterruptedException e1) { @@ -538,6 +564,38 @@ private static void saveFileTimeResolution(FileStore s, } } + private static String getConfigKey(FileStore s) { + final String storeKey; + if (SystemReader.getInstance().isWindows()) { + Object attribute = null; + try { + attribute = s.getAttribute("volume:vsn"); //$NON-NLS-1$ + } catch (IOException ignored) { + // ignore + } + if (attribute instanceof Integer) { + storeKey = attribute.toString(); + } else { + storeKey = s.name(); + } + } else { + storeKey = s.name(); + } + return javaVersionPrefix + storeKey; + } + + private static TimeUnit getUnit(long nanos) { + TimeUnit unit; + if (nanos < 200_000L) { + unit = TimeUnit.NANOSECONDS; + } else if (nanos < 200_000_000L) { + unit = TimeUnit.MICROSECONDS; + } else { + unit = TimeUnit.MILLISECONDS; + } + return unit; + } + private final @NonNull Duration fsTimestampResolution; private Duration minimalRacyInterval; @@ -565,7 +623,7 @@ public Duration getFsTimestampResolution() { * * @param fsTimestampResolution */ - public FileStoreAttributeCache( + public FileStoreAttributes( @NonNull Duration fsTimestampResolution) { this.fsTimestampResolution = fsTimestampResolution; this.minimalRacyInterval = Duration.ZERO; @@ -575,7 +633,7 @@ public FileStoreAttributeCache( @Override public String toString() { return String.format( - "FileStoreAttributeCache[fsTimestampResolution=%,d µs, " + "FileStoreAttributes[fsTimestampResolution=%,d µs, " + "minimalRacyInterval=%,d µs]", fsTimestampResolution.toNanos() / 1000, minimalRacyInterval.toNanos() / 1000); @@ -598,17 +656,16 @@ public static FS detect() { } /** - * Whether FileStore attribute cache entries should be determined - * asynchronously + * Whether FileStore attributes should be determined asynchronously * * @param asynch - * whether FileStore attribute cache entries should be determined + * whether FileStore attributes should be determined * asynchronously. If false access to cached attributes may block * for some seconds for the first call per FileStore * @since 5.1.9 */ - public static void setAsyncfileStoreAttrCache(boolean asynch) { - FileStoreAttributeCache.setBackground(asynch); + public static void setAsyncFileStoreAttributes(boolean asynch) { + FileStoreAttributes.setBackground(asynch); } /** @@ -639,9 +696,8 @@ public static FS detect(Boolean cygwinUsed) { } /** - * Get an estimate for the filesystem timestamp resolution from a cache of - * timestamp resolution per FileStore, if not yet available it is measured - * for a probe file under the given directory. + * Get cached FileStore attributes, if not yet available measure them using + * a probe file under the given directory. * * @param dir * the directory under which the probe file will be created to @@ -649,9 +705,9 @@ public static FS detect(Boolean cygwinUsed) { * @return measured filesystem timestamp resolution * @since 5.1.9 */ - public static FileStoreAttributeCache getFileStoreAttributeCache( + public static FileStoreAttributes getFileStoreAttributes( @NonNull Path dir) { - return FileStoreAttributeCache.get(dir); + return FileStoreAttributes.get(dir); } private volatile Holder userHome; From 275f3da783b025a3e6cfe47eedc7876e911269f3 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 5 Aug 2019 00:51:04 +0200 Subject: [PATCH 43/47] Fix FileSnapshot#save(long) and FileSnapshot#save(Instant) Use the fallback timestamp resolution as already described in the javadoc of these methods. Using zero file timestamp resolution doesn't make sense. Change-Id: Iaad2a0f99c3be3678e94980a0a368181b6aed38c Signed-off-by: Matthias Sohn --- .../eclipse/jgit/internal/storage/file/FileSnapshot.java | 8 ++++---- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 9 ++++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index e81a88451..77005100e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -44,7 +44,7 @@ package org.eclipse.jgit.internal.storage.file; import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_FILESTORE_ATTRIBUTES; - +import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_TIMESTAMP_RESOLUTION; import java.io.File; import java.io.IOException; import java.nio.file.attribute.BasicFileAttributes; @@ -176,7 +176,7 @@ private static Object getFileKey(BasicFileAttributes fileAttributes) { public static FileSnapshot save(long modified) { final Instant read = Instant.now(); return new FileSnapshot(read, Instant.ofEpochMilli(modified), - UNKNOWN_SIZE, Duration.ZERO, MISSING_FILEKEY); + UNKNOWN_SIZE, FALLBACK_TIMESTAMP_RESOLUTION, MISSING_FILEKEY); } /** @@ -196,8 +196,8 @@ public static FileSnapshot save(long modified) { */ public static FileSnapshot save(Instant modified) { final Instant read = Instant.now(); - return new FileSnapshot(read, modified, UNKNOWN_SIZE, Duration.ZERO, - MISSING_FILEKEY); + return new FileSnapshot(read, modified, UNKNOWN_SIZE, + FALLBACK_TIMESTAMP_RESOLUTION, MISSING_FILEKEY); } /** Last observed modification time of the path. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 16810e0dc..c30be36dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -213,13 +213,20 @@ public final static class FileStoreAttributes { private static final Duration UNDEFINED_DURATION = Duration .ofNanos(Long.MAX_VALUE); + /** + * Fallback filesystem timestamp resolution. The worst case timestamp + * resolution on FAT filesystems is 2 seconds. + */ + public static final Duration FALLBACK_TIMESTAMP_RESOLUTION = Duration + .ofMillis(2000); + /** * Fallback FileStore attributes used when we can't measure the * filesystem timestamp resolution. The last modified time granularity * of FAT filesystems is 2 seconds. */ public static final FileStoreAttributes FALLBACK_FILESTORE_ATTRIBUTES = new FileStoreAttributes( - Duration.ofMillis(2000)); + FALLBACK_TIMESTAMP_RESOLUTION); private static final Map attributeCache = new ConcurrentHashMap<>(); From 6857138e19ec6f597609339c3c88d58fb4dd0c0a Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 19 Jul 2019 17:35:27 +0200 Subject: [PATCH 44/47] Cache FileStoreAttributeCache per directory Cache FileStoreAttributeCache entries since looking up FileStore for a file may be expensive on some platforms. Implement a simple LRU cache based on ConcurrentHashMap using a simple long counter to order access to cache entries. Change-Id: I4881fa938ad2f17712c05da857838073a2fc4ddb Signed-off-by: Matthias Sohn Signed-off-by: Marc Strapetz Also-By: Marc Strapetz --- .../eclipse/jgit/util/SimpleLruCacheTest.java | 135 ++++++++++ org.eclipse.jgit/.settings/.api_filters | 8 + .../eclipse/jgit/internal/JGitText.properties | 1 + .../org/eclipse/jgit/internal/JGitText.java | 1 + .../src/org/eclipse/jgit/util/FS.java | 31 ++- .../org/eclipse/jgit/util/SimpleLruCache.java | 253 ++++++++++++++++++ 6 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/SimpleLruCacheTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/SimpleLruCache.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/SimpleLruCacheTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/SimpleLruCacheTest.java new file mode 100644 index 000000000..5894f7dc5 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/SimpleLruCacheTest.java @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2019, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class SimpleLruCacheTest { + + private Path trash; + + private SimpleLruCache cache; + + + @Before + public void setup() throws IOException { + trash = Files.createTempDirectory("tmp_"); + cache = new SimpleLruCache<>(100, 0.2f); + } + + @Before + @After + public void tearDown() throws Exception { + FileUtils.delete(trash.toFile(), + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); + } + + @Test + public void testPutGet() { + cache.put("a", "A"); + cache.put("z", "Z"); + assertEquals("A", cache.get("a")); + assertEquals("Z", cache.get("z")); + } + + @Test(expected = IllegalArgumentException.class) + public void testPurgeFactorTooLarge() { + cache.configure(5, 1.01f); + } + + @Test(expected = IllegalArgumentException.class) + public void testPurgeFactorTooLarge2() { + cache.configure(5, 100); + } + + @Test(expected = IllegalArgumentException.class) + public void testPurgeFactorTooSmall() { + cache.configure(5, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void testPurgeFactorTooSmall2() { + cache.configure(5, -100); + } + + @Test + public void testGetMissing() { + assertEquals(null, cache.get("a")); + } + + @Test + public void testPurge() { + for (int i = 0; i < 101; i++) { + cache.put("a" + i, "a" + i); + } + assertEquals(80, cache.size()); + assertNull(cache.get("a0")); + assertNull(cache.get("a20")); + assertNotNull(cache.get("a21")); + assertNotNull(cache.get("a99")); + } + + @Test + public void testConfigure() { + for (int i = 0; i < 100; i++) { + cache.put("a" + i, "a" + i); + } + assertEquals(100, cache.size()); + cache.configure(10, 0.3f); + assertEquals(7, cache.size()); + assertNull(cache.get("a0")); + assertNull(cache.get("a92")); + assertNotNull(cache.get("a93")); + assertNotNull(cache.get("a99")); + } +} diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 8277735e2..7aa7301b0 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -240,6 +240,14 @@ + + + + + + + + diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 6ac6955f0..290a0a28e 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -390,6 +390,7 @@ invalidPathContainsSeparator=Invalid path (contains separator ''{0}''): {1} invalidPathPeriodAtEndWindows=Invalid path (period at end is ignored by Windows): {0} invalidPathSpaceAtEndWindows=Invalid path (space at end is ignored by Windows): {0} invalidPathReservedOnWindows=Invalid path (''{0}'' is reserved on Windows): {1} +invalidPurgeFactor=Invalid purgeFactor {0}, values have to be in range between 0 and 1 invalidRedirectLocation=Invalid redirect location {0} -> {1} invalidRefAdvertisementLine=Invalid ref advertisement line: ''{1}'' invalidReflogRevision=Invalid reflog revision: {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 38c063d07..00aaa42b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -451,6 +451,7 @@ public static JGitText get() { /***/ public String invalidPathPeriodAtEndWindows; /***/ public String invalidPathSpaceAtEndWindows; /***/ public String invalidPathReservedOnWindows; + /***/ public String invalidPurgeFactor; /***/ public String invalidRedirectLocation; /***/ public String invalidRefAdvertisementLine; /***/ public String invalidReflogRevision; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index c30be36dd..19c04619d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -230,6 +230,9 @@ public final static class FileStoreAttributes { private static final Map attributeCache = new ConcurrentHashMap<>(); + private static final SimpleLruCache attrCacheByPath = new SimpleLruCache<>( + 100, 0.2f); + private static AtomicBoolean background = new AtomicBoolean(); private static Map locks = new ConcurrentHashMap<>(); @@ -245,6 +248,26 @@ private static void setBackground(boolean async) { private static final Duration FALLBACK_MIN_RACY_INTERVAL = Duration .ofMillis(10); + /** + * Configures size and purge factor of the path-based cache for file + * system attributes. Caching of file system attributes avoids recurring + * lookup of @{code FileStore} of files which may be expensive on some + * platforms. + * + * @param maxSize + * maximum size of the cache, default is 100 + * @param purgeFactor + * when the size of the map reaches maxSize the oldest + * entries will be purged to free up some space for new + * entries, {@code purgeFactor} is the fraction of + * {@code maxSize} to purge when this happens + * @since 5.1.9 + */ + public static void configureAttributesPathCache(int maxSize, + float purgeFactor) { + FileStoreAttributes.attrCacheByPath.configure(maxSize, purgeFactor); + } + /** * Get the FileStoreAttributes for the given FileStore * @@ -255,7 +278,13 @@ private static void setBackground(boolean async) { public static FileStoreAttributes get(Path path) { path = path.toAbsolutePath(); Path dir = Files.isDirectory(path) ? path : path.getParent(); - return getFileStoreAttributes(dir); + FileStoreAttributes cached = attrCacheByPath.get(dir); + if (cached != null) { + return cached; + } + FileStoreAttributes attrs = getFileStoreAttributes(dir); + attrCacheByPath.put(dir, attrs); + return attrs; } private static FileStoreAttributes getFileStoreAttributes(Path dir) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SimpleLruCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SimpleLruCache.java new file mode 100644 index 000000000..709d9ee73 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SimpleLruCache.java @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2019, Marc Strapetz + * Copyright (C) 2019, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.internal.JGitText; + +/** + * Simple limited size cache based on ConcurrentHashMap purging entries in LRU + * order when reaching size limit + * + * @param + * the type of keys maintained by this cache + * @param + * the type of mapped values + * + * @since 5.1.9 + */ +public class SimpleLruCache { + + private static class Entry { + + private final K key; + + private final V value; + + // pseudo clock timestamp of the last access to this entry + private volatile long lastAccessed; + + private long lastAccessedSorting; + + Entry(K key, V value, long lastAccessed) { + this.key = key; + this.value = value; + this.lastAccessed = lastAccessed; + } + + void copyAccessTime() { + lastAccessedSorting = lastAccessed; + } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "Entry [lastAccessed=" + lastAccessed + ", key=" + key + + ", value=" + value + "]"; + } + } + + private Lock lock = new ReentrantLock(); + + private Map> map = new ConcurrentHashMap<>(); + + private volatile int maximumSize; + + private int purgeSize; + + // pseudo clock to implement LRU order of access to entries + private volatile long time = 0L; + + private static void checkPurgeFactor(float purgeFactor) { + if (purgeFactor <= 0 || purgeFactor >= 1) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().invalidPurgeFactor, + Float.valueOf(purgeFactor))); + } + } + + private static int purgeSize(int maxSize, float purgeFactor) { + return (int) ((1 - purgeFactor) * maxSize); + } + + /** + * Create a new cache + * + * @param maxSize + * maximum size of the cache, to reduce need for synchronization + * this is not a hard limit. The real size of the cache could be + * slightly above this maximum if multiple threads put new values + * concurrently + * @param purgeFactor + * when the size of the map reaches maxSize the oldest entries + * will be purged to free up some space for new entries, + * {@code purgeFactor} is the fraction of {@code maxSize} to + * purge when this happens + */ + public SimpleLruCache(int maxSize, float purgeFactor) { + checkPurgeFactor(purgeFactor); + this.maximumSize = maxSize; + this.purgeSize = purgeSize(maxSize, purgeFactor); + } + + /** + * Returns the value to which the specified key is mapped, or {@code null} + * if this map contains no mapping for the key. + * + *

+ * More formally, if this cache contains a mapping from a key {@code k} to a + * value {@code v} such that {@code key.equals(k)}, then this method returns + * {@code v}; otherwise it returns {@code null}. (There can be at most one + * such mapping.) + * + * @param key + * the key + * + * @throws NullPointerException + * if the specified key is null + * + * @return value mapped for this key, or {@code null} if no value is mapped + */ + public V get(Object key) { + Entry entry = map.get(key); + if (entry != null) { + entry.lastAccessed = ++time; + return entry.value; + } + return null; + } + + /** + * Maps the specified key to the specified value in this cache. Neither the + * key nor the value can be null. + * + *

+ * The value can be retrieved by calling the {@code get} method with a key + * that is equal to the original key. + * + * @param key + * key with which the specified value is to be associated + * @param value + * value to be associated with the specified key + * @return the previous value associated with {@code key}, or {@code null} + * if there was no mapping for {@code key} + * @throws NullPointerException + * if the specified key or value is null + */ + public V put(@NonNull K key, @NonNull V value) { + map.put(key, new Entry<>(key, value, ++time)); + if (map.size() > maximumSize) { + purge(); + } + return value; + } + + /** + * Returns the current size of this cache + * + * @return the number of key-value mappings in this cache + */ + public int size() { + return map.size(); + } + + /** + * Reconfigures the cache. If {@code maxSize} is reduced some entries will + * be purged. + * + * @param maxSize + * maximum size of the cache + * + * @param purgeFactor + * when the size of the map reaches maxSize the oldest entries + * will be purged to free up some space for new entries, + * {@code purgeFactor} is the fraction of {@code maxSize} to + * purge when this happens + */ + public void configure(int maxSize, float purgeFactor) { + lock.lock(); + try { + checkPurgeFactor(purgeFactor); + this.maximumSize = maxSize; + this.purgeSize = purgeSize(maxSize, purgeFactor); + if (map.size() >= maximumSize) { + purge(); + } + } finally { + lock.unlock(); + } + } + + private void purge() { + // don't try to compete if another thread already has the lock + if (lock.tryLock()) { + try { + List entriesToPurge = new ArrayList<>(map.values()); + // copy access times to avoid other threads interfere with + // sorting + for (Entry e : entriesToPurge) { + e.copyAccessTime(); + } + Collections.sort(entriesToPurge, + Comparator.comparingLong(o -> -o.lastAccessedSorting)); + for (int index = purgeSize; index < entriesToPurge + .size(); index++) { + map.remove(entriesToPurge.get(index).key); + } + } finally { + lock.unlock(); + } + } + } +} From 3b368d5578db13b52b2485b11bf3da1e24ccffd2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 5 Aug 2019 18:00:35 +0200 Subject: [PATCH 45/47] In LockFile#waitForStatChange wait in units of file time resolution Since we now measure file time resolution we can use it to replace the hard coded wait time of 25ms. FileSnapshot#equals will return true until the mtime of the old (o) and the new FileSnapshot (n) differ by at least one file time resolution. Change-Id: Icb713a80ce9eb929242ed083406bfb6650c72223 Signed-off-by: Matthias Sohn --- .../src/org/eclipse/jgit/internal/storage/file/LockFile.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java index ccca0279a..420e73754 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java @@ -61,6 +61,7 @@ import java.nio.file.attribute.FileTime; import java.text.MessageFormat; import java.time.Instant; +import java.util.concurrent.TimeUnit; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; @@ -425,8 +426,10 @@ public void setFSync(boolean on) { public void waitForStatChange() throws InterruptedException { FileSnapshot o = FileSnapshot.save(ref); FileSnapshot n = FileSnapshot.save(lck); + long fsTimeResolution = FS.getFileStoreAttributes(lck.toPath()) + .getFsTimestampResolution().toNanos(); while (o.equals(n)) { - Thread.sleep(25 /* milliseconds */); + TimeUnit.NANOSECONDS.sleep(fsTimeResolution); try { Files.setLastModifiedTime(lck.toPath(), FileTime.from(Instant.now())); From e60b9e1879f8774e1afe07be4224605045f49eec Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 31 Jul 2019 17:27:47 +0200 Subject: [PATCH 46/47] FileSnapshot: fix bug with timestamp thresholding Increase the safety factor to 2.5x for extra safety if max of measured timestamp resolution and measured minimal racy threshold is < 100ms, use 1.25 otherwise since for large filesystem resolution values the influence of finite resolution of the system clock should be negligible. Before, not yet using the newly introduced minRacyThreshold measurement, the threshold was 1.1x FS resolution, and we could issue the following sequence of events, start create-file read-file (currentTime) end which had the following timestamps: create-file 1564589081998 start 1564589082002 read 1564589082003 end 1564589082004 In this case, the difference between create-file and read is 5ms, which exceeded the 4ms FS resolution, even though the events together took just 2ms of runtime. Reproduce with: bazel test --runs_per_test=100 \ //org.eclipse.jgit.test:org_eclipse_jgit_internal_storage_file_FileSnapshotTest The file system timestamp resolution is 4ms in this case. This code assumes that the kernel and the JVM use the same clock that is synchronized with the file system clock. This seems plausible, given the resolution of System.currentTimeMillis() and the latency for a gettimeofday system call (typically ~1us), but it would be good to justify this with specifications. Also cover a source of flakiness: if the test runs under extreme load, then we could have start create-file read end which would register as an unmodified file. Avoid this by skipping the test if end-start is too big. [msohn]: - downported from master to stable-5.1 - skip test if resolution is below 10ms - adjust safety factor to 1.25 for resolutions above 100ms Change-Id: I87d2cf035e01c44b7ba8364c410a860aa8e312ef Signed-off-by: Han-Wen Nienhuys Signed-off-by: Matthias Sohn --- .../storage/file/FileSnapshotTest.java | 36 ++++++++++++++----- .../internal/storage/file/FileSnapshot.java | 5 +-- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java index 40af9e2a0..6fa35d64b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileSnapshotTest.java @@ -47,6 +47,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; @@ -150,14 +151,33 @@ public void testNewFileWithWait() throws Exception { */ @Test public void testNewFileNoWait() throws Exception { - // if filesystem timestamp resolution is high the snapshot won't be - // racily clean - Assume.assumeTrue( - fsAttrCache.getFsTimestampResolution() - .compareTo(Duration.ofMillis(10)) > 0); - Path f1 = createFile("newfile"); - FileSnapshot save = FileSnapshot.save(f1.toFile()); - assertTrue(save.isModified(f1.toFile())); + // if filesystem timestamp resolution is smaller than time needed to + // create a file and FileSnapshot the snapshot won't be racily clean + Assume.assumeTrue(fsAttrCache.getFsTimestampResolution() + .compareTo(Duration.ofMillis(10)) > 0); + for (int i = 0; i < 50; i++) { + Instant start = Instant.now(); + Path f1 = createFile("newfile"); + FileSnapshot save = FileSnapshot.save(f1.toFile()); + Duration res = FS.getFileStoreAttributes(f1) + .getFsTimestampResolution(); + Instant end = Instant.now(); + if (Duration.between(start, end) + .compareTo(res.multipliedBy(2)) > 0) { + // This test is racy: under load, there may be a delay between createFile() and + // FileSnapshot.save(). This can stretch the time between the read TS and FS + // creation TS to the point that it exceeds the FS granularity, and we + // conclude it cannot be racily clean, and therefore must be really clean. + // + // This should be relatively uncommon. + continue; + } + // The file wasn't really modified, but it looks just like a "maybe racily clean" + // file. + assertTrue(save.isModified(f1.toFile())); + return; + } + fail("too much load for this test"); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 77005100e..976f946e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -523,8 +523,9 @@ private long getEffectiveRacyThreshold() { .getFsTimestampResolution().toNanos(); long minRacyInterval = fileStoreAttributeCache.getMinimalRacyInterval() .toNanos(); - // add a 30% safety margin - return Math.max(timestampResolution, minRacyInterval) * 13 / 10; + long max = Math.max(timestampResolution, minRacyInterval); + // safety margin: factor 2.5 below 100ms otherwise 1.25 + return max < 100_000_000L ? max * 5 / 2 : max * 5 / 4; } private boolean isModified(Instant currLastModified) { From 5a88815b1ceb3db7eaf46d22f20fa20270f6f7c4 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 8 Aug 2019 10:10:12 +0200 Subject: [PATCH 47/47] Fix OpenSshConfigTest#config - use FS.DETECTED instead of db.getFS() since the ssh config is typically in a different place than the repository, the same is used in OpenSshConfig - reduce unnecessary repeated writes by introducing wait for one tick of the file time resolution Change-Id: Ifac915e97ff420ec5cf8e2f162e351f9f51b6b14 Signed-off-by: Matthias Sohn --- .../tst/org/eclipse/jgit/transport/OpenSshConfigTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java index 7777a3c99..0358718cf 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/OpenSshConfigTest.java @@ -58,6 +58,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.time.Instant; +import java.util.concurrent.TimeUnit; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.Constants; @@ -93,12 +94,17 @@ public void setUp() throws Exception { } private void config(String data) throws IOException { - FS fs = db.getFS(); + FS fs = FS.DETECTED; + long resolution = FS.getFileStoreAttributes(configFile.toPath()) + .getFsTimestampResolution().toNanos(); Instant lastMtime = fs.lastModifiedInstant(configFile); do { try (final OutputStreamWriter fw = new OutputStreamWriter( new FileOutputStream(configFile), UTF_8)) { fw.write(data); + TimeUnit.NANOSECONDS.sleep(resolution); + } catch (InterruptedException e) { + Thread.interrupted(); } } while (lastMtime.equals(fs.lastModifiedInstant(configFile))); }