diff --git a/.bazelversion b/.bazelversion index 1545d9665..fcdb2e109 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -3.5.0 +4.0.0 diff --git a/BUILD b/BUILD index be6dd767d..184ab272d 100644 --- a/BUILD +++ b/BUILD @@ -13,6 +13,8 @@ genrule( "//org.eclipse.jgit.lfs:jgit-lfs", "//org.eclipse.jgit.lfs.server:jgit-lfs-server", "//org.eclipse.jgit.junit:junit", + "//org.eclipse.jgit.ssh.apache:ssh-apache", + "//org.eclipse.jgit.ssh.jsch:ssh-jsch", ], outs = ["all.zip"], cmd = " && ".join([ diff --git a/WORKSPACE b/WORKSPACE index 7651cba4e..224968a26 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -27,6 +27,34 @@ load( "maven_jar", ) +http_archive( + name = "openjdk15_linux_archive", + build_file_content = """ +java_runtime(name = 'runtime', srcs = glob(['**']), visibility = ['//visibility:public']) +exports_files(["WORKSPACE"], visibility = ["//visibility:public"]) +""", + sha256 = "0a38f1138c15a4f243b75eb82f8ef40855afcc402e3c2a6de97ce8235011b1ad", + strip_prefix = "zulu15.27.17-ca-jdk15.0.0-linux_x64", + urls = [ + "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu15.27.17-ca-jdk15.0.0-linux_x64.tar.gz", + "https://cdn.azul.com/zulu/bin/zulu15.27.17-ca-jdk15.0.0-linux_x64.tar.gz", + ], +) + +http_archive( + name = "openjdk15_darwin_archive", + build_file_content = """ +java_runtime(name = 'runtime', srcs = glob(['**']), visibility = ['//visibility:public']) +exports_files(["WORKSPACE"], visibility = ["//visibility:public"]) +""", + sha256 = "f80b2e0512d9d8a92be24497334c974bfecc8c898fc215ce0e76594f00437482", + strip_prefix = "zulu15.27.17-ca-jdk15.0.0-macosx_x64", + urls = [ + "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu15.27.17-ca-jdk15.0.0-macosx_x64.tar.gz", + "https://cdn.azul.com/zulu/bin/zulu15.27.17-ca-jdk15.0.0-macosx_x64.tar.gz", + ], +) + JMH_VERS = "1.21" maven_jar( @@ -83,26 +111,26 @@ maven_jar( maven_jar( name = "httpclient", - artifact = "org.apache.httpcomponents:httpclient:4.5.10", - sha1 = "7ca2e4276f4ef95e4db725a8cd4a1d1e7585b9e5", + artifact = "org.apache.httpcomponents:httpclient:4.5.13", + sha1 = "e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada", ) maven_jar( name = "httpcore", - artifact = "org.apache.httpcomponents:httpcore:4.4.12", - sha1 = "21ebaf6d532bc350ba95bd81938fa5f0e511c132", + artifact = "org.apache.httpcomponents:httpcore:4.4.14", + sha1 = "9dd1a631c082d92ecd4bd8fd4cf55026c720a8c1", ) maven_jar( name = "sshd-osgi", - artifact = "org.apache.sshd:sshd-osgi:2.4.0", - sha1 = "fc4551c1eeda35e4671b263297d37d2bca81c4d4", + artifact = "org.apache.sshd:sshd-osgi:2.6.0", + sha1 = "40e365bb799e1bff3d31dc858b1e59a93c123f29", ) maven_jar( name = "sshd-sftp", - artifact = "org.apache.sshd:sshd-sftp:2.4.0", - sha1 = "92e1b7d1e19c715efb4a8871d34145da8f87cdb2", + artifact = "org.apache.sshd:sshd-sftp:2.6.0", + sha1 = "6eddfe8fdf59a3d9a49151e4177f8c1bebeb30c9", ) maven_jar( @@ -205,52 +233,59 @@ maven_jar( maven_jar( name = "gson", - artifact = "com.google.code.gson:gson:2.8.2", - sha1 = "3edcfe49d2c6053a70a2a47e4e1c2f94998a49cf", + artifact = "com.google.code.gson:gson:2.8.6", + sha1 = "9180733b7df8542621dc12e21e87557e8c99b8cb", ) -JETTY_VER = "9.4.30.v20200611" +JETTY_VER = "9.4.36.v20210114" maven_jar( name = "jetty-servlet", artifact = "org.eclipse.jetty:jetty-servlet:" + JETTY_VER, - sha1 = "ca3dea2cd34ee88cec017001603af0c9e74781d6", - src_sha1 = "6908f24428060bd542bddfa3e89e03d0dbbc2a6d", + sha1 = "b189e52a5ee55ae172e4e99e29c5c314f5daf4b9", + src_sha1 = "3a0fa449772ab0d76625f6afb81f60c32a490613", ) maven_jar( name = "jetty-security", artifact = "org.eclipse.jetty:jetty-security:" + JETTY_VER, - sha1 = "1a5261f6ad4081ad9e9bb01416d639931d391273", - src_sha1 = "6ca41b34aa4f84c267603edd4b069122bd5f17d3", + sha1 = "42030d6ed7dfc0f75818cde0adcf738efc477574", + src_sha1 = "612220a97d45fad3983ccc56b0cb9a271f3fd003", ) maven_jar( name = "jetty-server", artifact = "org.eclipse.jetty:jetty-server:" + JETTY_VER, - sha1 = "e5ede3724d062717d0c04e4c77f74fe8115c2a6f", - src_sha1 = "c8b02a47a35c1f083b310cbd202738cf08bc1d55", + sha1 = "88a7d342974aadca658e7386e8d0fcc5c0788f41", + src_sha1 = "4552c0c6db2948e8557db477b6b48d291006e481", ) maven_jar( name = "jetty-http", artifact = "org.eclipse.jetty:jetty-http:" + JETTY_VER, - sha1 = "cd6223382e4f82b9ea807d8cdb04a23e5d629f1c", - src_sha1 = "00520c04b10609b981159b5ca284b5a158c077a9", + sha1 = "1eee89a55e04ff94df0f85d95200fc48acb43d86", + src_sha1 = "552a784ec789c7ba581c5341ae6d8b6353ed5ace", ) maven_jar( name = "jetty-io", artifact = "org.eclipse.jetty:jetty-io:" + JETTY_VER, - sha1 = "9c360d08e903b2dbd5d1f8e889a32046948628ce", - src_sha1 = "dac8f8a3f84afdd3686d36f58b5ccb276961b8ce", + sha1 = "84a8faf9031eb45a5a2ddb7681e22c483d81ab3a", + src_sha1 = "72d5fc6d909e28f8720394b25babda80805a46b9", ) maven_jar( name = "jetty-util", artifact = "org.eclipse.jetty:jetty-util:" + JETTY_VER, - sha1 = "39ec6aa4745952077f5407cb1394d8ba2db88b13", - src_sha1 = "f41f9391f91884a79350f3ad9b09b8e46c9be0ec", + sha1 = "925257fbcca6b501a25252c7447dbedb021f7404", + src_sha1 = "532e8b66044f4e58ca5da3aec19f02a2f3c16ddd", +) + +maven_jar( + name = "jetty-util-ajax", + artifact = "org.eclipse.jetty:jetty-util-ajax:" + JETTY_VER, + sha1 = "2f478130c21787073facb64d7242e06f94980c60", + src_sha1 = "7153d7ca38878d971fd90992c303bb7719ba7a21", ) BOUNCYCASTLE_VER = "1.65" diff --git a/lib/BUILD b/lib/BUILD index 7720696b3..8ad49d450 100644 --- a/lib/BUILD +++ b/lib/BUILD @@ -132,7 +132,10 @@ java_library( name = "jetty-servlet", # TODO: This should be testonly but org.eclipse.jgit.pgm depends on it. visibility = ["//visibility:public"], - exports = ["@jetty-servlet//jar"], + exports = [ + "@jetty-servlet//jar", + "@jetty-util-ajax//jar", + ], ) java_library( @@ -159,6 +162,7 @@ java_library( "//org.eclipse.jgit:__pkg__", "//org.eclipse.jgit.gpg.bc:__pkg__", "//org.eclipse.jgit.test:__pkg__", + "//org.eclipse.jgit.gpg.bc.test:__pkg__", ], exports = ["@bcpg//jar"], ) @@ -169,6 +173,7 @@ java_library( "//org.eclipse.jgit:__pkg__", "//org.eclipse.jgit.gpg.bc:__pkg__", "//org.eclipse.jgit.test:__pkg__", + "//org.eclipse.jgit.gpg.bc.test:__pkg__", ], exports = ["@bcprov//jar"], ) diff --git a/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..b853c6a7e 100644 --- a/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.ant/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit.ant/META-INF/SOURCE-MANIFEST.MF index 7ab4b3e0a..e5aeb3142 100644 --- a/org.eclipse.jgit.ant/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit.ant/META-INF/SOURCE-MANIFEST.MF @@ -5,3 +5,4 @@ Bundle-SymbolicName: org.eclipse.jgit.ant.source Bundle-Vendor: Eclipse.org - JGit Bundle-Version: 6.0.0.qualifier Eclipse-SourceBundle: org.eclipse.jgit.ant;version="6.0.0.qualifier";roots="." + diff --git a/org.eclipse.jgit.archive/.settings/.api_filters b/org.eclipse.jgit.archive/.settings/.api_filters new file mode 100644 index 000000000..f4a934aeb --- /dev/null +++ b/org.eclipse.jgit.archive/.settings/.api_filters @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/org.eclipse.jgit.archive/META-INF/MANIFEST.MF b/org.eclipse.jgit.archive/META-INF/MANIFEST.MF index 1441f62e1..428c0a57f 100644 --- a/org.eclipse.jgit.archive/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.archive/META-INF/MANIFEST.MF @@ -27,3 +27,4 @@ Export-Package: org.eclipse.jgit.archive;version="6.0.0"; org.apache.commons.compress.archivers, org.osgi.framework", org.eclipse.jgit.archive.internal;version="6.0.0";x-internal:=true + diff --git a/org.eclipse.jgit.archive/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit.archive/META-INF/SOURCE-MANIFEST.MF index 7322fd082..d5cda97e9 100644 --- a/org.eclipse.jgit.archive/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit.archive/META-INF/SOURCE-MANIFEST.MF @@ -5,3 +5,4 @@ Bundle-SymbolicName: org.eclipse.jgit.archive.source Bundle-Vendor: Eclipse.org - JGit Bundle-Version: 6.0.0.qualifier Eclipse-SourceBundle: org.eclipse.jgit.archive;version="6.0.0.qualifier";roots="." + diff --git a/org.eclipse.jgit.archive/resources/org/eclipse/jgit/archive/internal/ArchiveText.properties b/org.eclipse.jgit.archive/resources/org/eclipse/jgit/archive/internal/ArchiveText.properties index 3b50bb4fd..e6e122734 100644 --- a/org.eclipse.jgit.archive/resources/org/eclipse/jgit/archive/internal/ArchiveText.properties +++ b/org.eclipse.jgit.archive/resources/org/eclipse/jgit/archive/internal/ArchiveText.properties @@ -1,3 +1,4 @@ cannotSetOption=Cannot set option: {0} +invalidCompressionLevel=Invalid compression level: {0} pathDoesNotMatchMode=Path {0} does not match mode {1} unsupportedMode=Unsupported mode {0} diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/BaseFormat.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/BaseFormat.java index 27f001e22..0ebac7722 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/BaseFormat.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/BaseFormat.java @@ -25,6 +25,11 @@ * @since 4.0 */ public class BaseFormat { + /** + * Compression-level for the archive file. Only values in [0-9] are allowed. + * @since 5.11 + */ + protected static final String COMPRESSION_LEVEL = "compression-level"; //$NON-NLS-1$ /** * Apply options to archive output stream @@ -40,6 +45,9 @@ protected ArchiveOutputStream applyFormatOptions(ArchiveOutputStream s, Map o) throws IOException { for (Map.Entry p : o.entrySet()) { try { + if (p.getKey().equals(COMPRESSION_LEVEL)) { + continue; + } new Statement(s, "set" + StringUtils.capitalize(p.getKey()), //$NON-NLS-1$ new Object[] { p.getValue() }).execute(); } catch (Exception e) { @@ -49,4 +57,32 @@ protected ArchiveOutputStream applyFormatOptions(ArchiveOutputStream s, } return s; } + + /** + * Removes and returns the {@link #COMPRESSION_LEVEL} key from the input map + * parameter if it exists, or -1 if this key does not exist. + * + * @param o + * options map + * @return The compression level if it exists in the map, or -1 instead. + * @throws IllegalArgumentException + * if the {@link #COMPRESSION_LEVEL} option does not parse to an + * Integer. + * @since 5.11 + */ + protected int getCompressionLevel(Map o) { + if (!o.containsKey(COMPRESSION_LEVEL)) { + return -1; + } + Object option = o.get(COMPRESSION_LEVEL); + try { + Integer compressionLevel = (Integer) option; + return compressionLevel.intValue(); + } catch (ClassCastException e) { + throw new IllegalArgumentException( + MessageFormat.format( + ArchiveText.get().invalidCompressionLevel, option), + e); + } + } } diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/Tbz2Format.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/Tbz2Format.java index e880f5ec5..940dafd40 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/Tbz2Format.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/Tbz2Format.java @@ -45,7 +45,13 @@ public ArchiveOutputStream createArchiveOutputStream(OutputStream s) @Override public ArchiveOutputStream createArchiveOutputStream(OutputStream s, Map o) throws IOException { - BZip2CompressorOutputStream out = new BZip2CompressorOutputStream(s); + BZip2CompressorOutputStream out; + int compressionLevel = getCompressionLevel(o); + if (compressionLevel != -1) { + out = new BZip2CompressorOutputStream(s, compressionLevel); + } else { + out = new BZip2CompressorOutputStream(s); + } return tarFormat.createArchiveOutputStream(out, o); } diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TgzFormat.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TgzFormat.java index 859a59d09..72e2439f6 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TgzFormat.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TgzFormat.java @@ -18,6 +18,7 @@ import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.eclipse.jgit.api.ArchiveCommand; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; @@ -45,7 +46,15 @@ public ArchiveOutputStream createArchiveOutputStream(OutputStream s) @Override public ArchiveOutputStream createArchiveOutputStream(OutputStream s, Map o) throws IOException { - GzipCompressorOutputStream out = new GzipCompressorOutputStream(s); + GzipCompressorOutputStream out; + int compressionLevel = getCompressionLevel(o); + if (compressionLevel != -1) { + GzipParameters parameters = new GzipParameters(); + parameters.setCompressionLevel(compressionLevel); + out = new GzipCompressorOutputStream(s, parameters); + } else { + out = new GzipCompressorOutputStream(s); + } return tarFormat.createArchiveOutputStream(out, o); } diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TxzFormat.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TxzFormat.java index 484ab5775..b16fb6dcb 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TxzFormat.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/TxzFormat.java @@ -45,7 +45,13 @@ public ArchiveOutputStream createArchiveOutputStream(OutputStream s) @Override public ArchiveOutputStream createArchiveOutputStream(OutputStream s, Map o) throws IOException { - XZCompressorOutputStream out = new XZCompressorOutputStream(s); + XZCompressorOutputStream out; + int compressionLevel = getCompressionLevel(o); + if (compressionLevel != -1) { + out = new XZCompressorOutputStream(s, compressionLevel); + } else { + out = new XZCompressorOutputStream(s); + } return tarFormat.createArchiveOutputStream(out, o); } diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/ZipFormat.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/ZipFormat.java index 59a9765f2..97a24c75c 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/ZipFormat.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/ZipFormat.java @@ -47,7 +47,12 @@ public ArchiveOutputStream createArchiveOutputStream(OutputStream s) @Override public ArchiveOutputStream createArchiveOutputStream(OutputStream s, Map o) throws IOException { - return applyFormatOptions(new ZipArchiveOutputStream(s), o); + ZipArchiveOutputStream out = new ZipArchiveOutputStream(s); + int compressionLevel = getCompressionLevel(o); + if (compressionLevel != -1) { + out.setLevel(compressionLevel); + } + return applyFormatOptions(out, o); } /** {@inheritDoc} */ diff --git a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/internal/ArchiveText.java b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/internal/ArchiveText.java index 45f96fa4c..551646bcd 100644 --- a/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/internal/ArchiveText.java +++ b/org.eclipse.jgit.archive/src/org/eclipse/jgit/archive/internal/ArchiveText.java @@ -28,6 +28,7 @@ public static ArchiveText get() { // @formatter:off /***/ public String cannotSetOption; + /***/ public String invalidCompressionLevel; /***/ public String pathDoesNotMatchMode; /***/ public String unsupportedMode; } diff --git a/org.eclipse.jgit.benchmarks/.settings/edu.umd.cs.findbugs.core.prefs b/org.eclipse.jgit.benchmarks/.settings/edu.umd.cs.findbugs.core.prefs new file mode 100644 index 000000000..1c0a3448a --- /dev/null +++ b/org.eclipse.jgit.benchmarks/.settings/edu.umd.cs.findbugs.core.prefs @@ -0,0 +1,145 @@ +#SpotBugs User Preferences +#Fri Dec 04 10:39:51 CET 2020 +detectorExplicitSerialization=ExplicitSerialization|true +detectorMultithreadedInstanceAccess=MultithreadedInstanceAccess|true +detectorConfusionBetweenInheritedAndOuterMethod=ConfusionBetweenInheritedAndOuterMethod|true +detectorWrongMapIterator=WrongMapIterator|true +detectorUnnecessaryMath=UnnecessaryMath|true +detectorUselessSubclassMethod=UselessSubclassMethod|false +filter_settings=Medium|BAD_PRACTICE,CORRECTNESS,EXPERIMENTAL,I18N,MALICIOUS_CODE,MT_CORRECTNESS,PERFORMANCE,SECURITY,STYLE|false|15 +detectorURLProblems=URLProblems|true +detectorIteratorIdioms=IteratorIdioms|true +detectorMutableEnum=MutableEnum|true +detectorFindNonShortCircuit=FindNonShortCircuit|true +detectorSynchronizeAndNullCheckField=SynchronizeAndNullCheckField|true +detectorVolatileUsage=VolatileUsage|true +detectorFindNakedNotify=FindNakedNotify|true +detectorFindUninitializedGet=FindUninitializedGet|true +detectorFindUseOfNonSerializableValue=FindUseOfNonSerializableValue|true +detectorFindJSR166LockMonitorenter=FindJSR166LockMonitorenter|true +detectorQuestionableBooleanAssignment=QuestionableBooleanAssignment|true +detectorSwitchFallthrough=SwitchFallthrough|true +detectorFindLocalSelfAssignment2=FindLocalSelfAssignment2|true +detectorConfusedInheritance=ConfusedInheritance|true +detectorSynchronizationOnSharedBuiltinConstant=SynchronizationOnSharedBuiltinConstant|true +detectorMutableStaticFields=MutableStaticFields|true +detectorInvalidJUnitTest=InvalidJUnitTest|true +detectorInfiniteLoop=InfiniteLoop|true +detectorFindRunInvocations=FindRunInvocations|true +detectorBadSyntaxForRegularExpression=BadSyntaxForRegularExpression|true +detectorXMLFactoryBypass=XMLFactoryBypass|true +detectorFindOpenStream=FindOpenStream|true +detectorCheckExpectedWarnings=CheckExpectedWarnings|false +detectorHugeSharedStringConstants=HugeSharedStringConstants|true +detectorLostLoggerDueToWeakReference=LostLoggerDueToWeakReference|true +detectorStringConcatenation=StringConcatenation|true +detectorLoadOfKnownNullValue=LoadOfKnownNullValue|true +detectorFinalizerNullsFields=FinalizerNullsFields|true +detectorFindFieldSelfAssignment=FindFieldSelfAssignment|true +detectorInefficientToArray=InefficientToArray|false +detectorDontCatchIllegalMonitorStateException=DontCatchIllegalMonitorStateException|true +detectorInconsistentAnnotations=InconsistentAnnotations|true +detectorBadlyOverriddenAdapter=BadlyOverriddenAdapter|true +detectorInstantiateStaticClass=InstantiateStaticClass|true +detectorCheckRelaxingNullnessAnnotation=CheckRelaxingNullnessAnnotation|true +detectorMethodReturnCheck=MethodReturnCheck|true +detectorEqualsOperandShouldHaveClassCompatibleWithThis=EqualsOperandShouldHaveClassCompatibleWithThis|true +detectorFindDoubleCheck=FindDoubleCheck|true +detectorFindBadForLoop=FindBadForLoop|true +detectorDefaultEncodingDetector=DefaultEncodingDetector|true +detectorFindInconsistentSync2=FindInconsistentSync2|true +detectorFindSpinLoop=FindSpinLoop|true +detectorFindMaskedFields=FindMaskedFields|true +detectorBooleanReturnNull=BooleanReturnNull|true +detectorFindUnsyncGet=FindUnsyncGet|true +detectorCrossSiteScripting=CrossSiteScripting|true +detectorDroppedException=DroppedException|true +detectorFindDeadLocalStores=FindDeadLocalStores|true +detectorCheckImmutableAnnotation=CheckImmutableAnnotation|true +detectorInfiniteRecursiveLoop=InfiniteRecursiveLoop|true +detectorFindRefComparison=FindRefComparison|true +detectorFindRoughConstants=FindRoughConstants|true +detectorMutableLock=MutableLock|true +detectorFindNullDeref=FindNullDeref|true +detectorFindReturnRef=FindReturnRef|true +detectorSynchronizeOnClassLiteralNotGetClass=SynchronizeOnClassLiteralNotGetClass|true +detectorFindUselessControlFlow=FindUselessControlFlow|true +detectorOverridingEqualsNotSymmetrical=OverridingEqualsNotSymmetrical|true +detectorIDivResultCastToDouble=IDivResultCastToDouble|true +detectorReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass=ReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass|true +detectorFindSelfComparison=FindSelfComparison|true +detectorFindFloatEquality=FindFloatEquality|true +detectorFindComparatorProblems=FindComparatorProblems|true +detectorRepeatedConditionals=RepeatedConditionals|true +filter_settings_neg=NOISE| +detectorInefficientMemberAccess=InefficientMemberAccess|false +detectorFindUncalledPrivateMethods=FindUncalledPrivateMethods|true +detectorNumberConstructor=NumberConstructor|true +detectorDontAssertInstanceofInTests=DontAssertInstanceofInTests|true +detectorFindFinalizeInvocations=FindFinalizeInvocations|true +detectorFindNullDerefsInvolvingNonShortCircuitEvaluation=FindNullDerefsInvolvingNonShortCircuitEvaluation|true +detectorDontIgnoreResultOfPutIfAbsent=DontIgnoreResultOfPutIfAbsent|true +detectorFindUnconditionalWait=FindUnconditionalWait|true +detectorFindTwoLockWait=FindTwoLockWait|true +detectorFindSleepWithLockHeld=FindSleepWithLockHeld|true +detectorFindUnreleasedLock=FindUnreleasedLock|true +detectorInefficientIndexOf=InefficientIndexOf|false +detectorDoInsideDoPrivileged=DoInsideDoPrivileged|true +detectorFindEmptySynchronizedBlock=FindEmptySynchronizedBlock|true +detectorOverridingMethodsMustInvokeSuperDetector=OverridingMethodsMustInvokeSuperDetector|true +detectorWaitInLoop=WaitInLoop|true +detectorIntCast2LongAsInstant=IntCast2LongAsInstant|true +detectorBadUseOfReturnValue=BadUseOfReturnValue|true +detectorFindSqlInjection=FindSqlInjection|true +detectorUnreadFields=UnreadFields|true +detectorSynchronizingOnContentsOfFieldToProtectField=SynchronizingOnContentsOfFieldToProtectField|true +detectorFindUselessObjects=FindUselessObjects|true +detectorBadAppletConstructor=BadAppletConstructor|false +detectorInheritanceUnsafeGetResource=InheritanceUnsafeGetResource|true +detectorSerializableIdiom=SerializableIdiom|true +detectorNaming=Naming|true +detectorNoteUnconditionalParamDerefs=NoteUnconditionalParamDerefs|true +detectorFormatStringChecker=FormatStringChecker|true +detectorSuspiciousThreadInterrupted=SuspiciousThreadInterrupted|true +detectorEmptyZipFileEntry=EmptyZipFileEntry|false +detectorFindCircularDependencies=FindCircularDependencies|false +detectorPreferZeroLengthArrays=PreferZeroLengthArrays|true +detectorAtomicityProblem=AtomicityProblem|true +detectorRuntimeExceptionCapture=RuntimeExceptionCapture|true +detectorInitializationChain=InitializationChain|true +detectorInitializeNonnullFieldsInConstructor=InitializeNonnullFieldsInConstructor|true +detectorOptionalReturnNull=OptionalReturnNull|true +detectorStartInConstructor=StartInConstructor|true +detectorFindUnsatisfiedObligation=FindUnsatisfiedObligation|true +detectorRedundantConditions=RedundantConditions|true +effort=default +detectorRedundantInterfaces=RedundantInterfaces|true +detectorDuplicateBranches=DuplicateBranches|true +detectorCheckTypeQualifiers=CheckTypeQualifiers|true +detectorComparatorIdiom=ComparatorIdiom|true +detectorFindBadCast2=FindBadCast2|true +detectorFindMismatchedWaitOrNotify=FindMismatchedWaitOrNotify|true +excludefilter0=findBugs/FindBugsExcludeFilter.xml|true +detectorBadResultSetAccess=BadResultSetAccess|true +detectorIncompatMask=IncompatMask|true +detectorCovariantArrayAssignment=CovariantArrayAssignment|false +detectorDumbMethodInvocations=DumbMethodInvocations|true +run_at_full_build=false +detectorStaticCalendarDetector=StaticCalendarDetector|true +detectorUncallableMethodOfAnonymousClass=UncallableMethodOfAnonymousClass|true +detectorVarArgsProblems=VarArgsProblems|true +detectorInefficientInitializationInsideLoop=InefficientInitializationInsideLoop|false +detectorCloneIdiom=CloneIdiom|true +detectorFindHEmismatch=FindHEmismatch|true +detectorAppendingToAnObjectOutputStream=AppendingToAnObjectOutputStream|true +detectorFindSelfComparison2=FindSelfComparison2|true +detectorLazyInit=LazyInit|true +detectorFindUnrelatedTypesInGenericContainer=FindUnrelatedTypesInGenericContainer|true +detectorDontUseEnum=DontUseEnum|true +detectorFindPuzzlers=FindPuzzlers|true +detectorCallToUnsupportedMethod=CallToUnsupportedMethod|false +detectorSuperfluousInstanceOf=SuperfluousInstanceOf|true +detectorReadReturnShouldBeChecked=ReadReturnShouldBeChecked|true +detector_threshold=2 +detectorPublicSemaphores=PublicSemaphores|false +detectorDumbMethods=DumbMethods|true diff --git a/org.eclipse.jgit.benchmarks/findBugs/FindBugsExcludeFilter.xml b/org.eclipse.jgit.benchmarks/findBugs/FindBugsExcludeFilter.xml new file mode 100644 index 000000000..ad63e8f87 --- /dev/null +++ b/org.eclipse.jgit.benchmarks/findBugs/FindBugsExcludeFilter.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/org.eclipse.jgit.benchmarks/pom.xml b/org.eclipse.jgit.benchmarks/pom.xml index 5941f1364..e5954e54d 100644 --- a/org.eclipse.jgit.benchmarks/pom.xml +++ b/org.eclipse.jgit.benchmarks/pom.xml @@ -49,6 +49,26 @@ src/ + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + enforce-maven + + enforce + + + + + 3.6.3 + + + + + + org.apache.maven.plugins maven-compiler-plugin @@ -95,7 +115,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.2.4 package @@ -157,19 +177,19 @@ org.apache.maven.plugins maven-site-plugin - 3.8.2 + 3.9.1 org.apache.maven.wagon wagon-ssh - 3.3.4 + 3.4.2 org.apache.maven.plugins maven-surefire-report-plugin - 3.0.0-M3 + 3.0.0-M5 org.apache.maven.plugins diff --git a/org.eclipse.jgit.gpg.bc.test/.classpath b/org.eclipse.jgit.gpg.bc.test/.classpath index f08af0a4e..0acccbaae 100644 --- a/org.eclipse.jgit.gpg.bc.test/.classpath +++ b/org.eclipse.jgit.gpg.bc.test/.classpath @@ -2,10 +2,15 @@ - + - + + + + + + diff --git a/org.eclipse.jgit.gpg.bc.test/.gitignore b/org.eclipse.jgit.gpg.bc.test/.gitignore index 934e0e06f..8b6760c93 100644 --- a/org.eclipse.jgit.gpg.bc.test/.gitignore +++ b/org.eclipse.jgit.gpg.bc.test/.gitignore @@ -1,2 +1,3 @@ /bin +/bin-tst /target diff --git a/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs index 822846c4d..cba893f04 100644 --- a/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.gpg.bc.test/BUILD b/org.eclipse.jgit.gpg.bc.test/BUILD index 1e3677d92..925536e5d 100644 --- a/org.eclipse.jgit.gpg.bc.test/BUILD +++ b/org.eclipse.jgit.gpg.bc.test/BUILD @@ -1,3 +1,8 @@ +load( + "@com_googlesource_gerrit_bazlets//tools:genrule2.bzl", + "genrule2", +) +load("@rules_java//java:defs.bzl", "java_import") load( "@com_googlesource_gerrit_bazlets//tools:junit.bzl", "junit_tests", @@ -8,7 +13,23 @@ junit_tests( srcs = glob(["tst/**/*.java"]), tags = ["bc"], deps = [ + "//lib:bcpg", + "//lib:bcprov", "//lib:junit", + "//org.eclipse.jgit:jgit", "//org.eclipse.jgit.gpg.bc:gpg-bc", + "//org.eclipse.jgit.gpg.bc.test:tst_rsrc", ], ) + +java_import( + name = "tst_rsrc", + jars = [":tst_rsrc_jar"], +) + +genrule2( + name = "tst_rsrc_jar", + srcs = glob(["tst-rsrc/**"]), + outs = ["tst_rsrc.jar"], + cmd = "o=$$PWD/$@ && tar cf - $(SRCS) | tar -C $$TMP --strip-components=2 -xf - && cd $$TMP && zip -qr $$o .", +) diff --git a/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF index 727e0cbc3..187c583fa 100644 --- a/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.gpg.bc.test/META-INF/MANIFEST.MF @@ -7,8 +7,18 @@ Bundle-Version: 6.0.0.qualifier Bundle-Vendor: %Bundle-Vendor Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.eclipse.jgit.gpg.bc.internal;version="[6.0.0,6.1.0)", - org.junit;version="[4.13,5.0.0)" -Export-Package: org.eclipse.jgit.gpg.bc.internal;x-internal:=true +Import-Package: org.bouncycastle.jce.provider;version="[1.65.0,2.0.0)", + org.bouncycastle.openpgp;version="[1.65.0,2.0.0)", + org.bouncycastle.openpgp.operator;version="[1.65.0,2.0.0)", + org.bouncycastle.openpgp.operator.jcajce;version="[1.65.0,2.0.0)", + org.bouncycastle.util.encoders;version="[1.65.0,2.0.0)", + org.eclipse.jgit.gpg.bc.internal;version="[6.0.0,6.1.0)", + org.eclipse.jgit.gpg.bc.internal.keys;version="[6.0.0,6.1.0)", + org.eclipse.jgit.util.sha1;version="[6.0.0,6.1.0)", + org.junit;version="[4.13,5.0.0)", + org.junit.runner;version="[4.13,5.0.0)", + org.junit.runners;version="[4.13,5.0.0)" +Export-Package: org.eclipse.jgit.gpg.bc.internal;x-internal:=true, + org.eclipse.jgit.gpg.bc.internal.keys;x-internal:=true Require-Bundle: org.hamcrest.core;bundle-version="[1.1.0,2.0.0)", org.hamcrest.library;bundle-version="[1.1.0,2.0.0)" diff --git a/org.eclipse.jgit.gpg.bc.test/build.properties b/org.eclipse.jgit.gpg.bc.test/build.properties index 9ffa0caf7..e36d6667e 100644 --- a/org.eclipse.jgit.gpg.bc.test/build.properties +++ b/org.eclipse.jgit.gpg.bc.test/build.properties @@ -1,5 +1,5 @@ source.. = tst/ -output.. = bin/ +output.. = bin-tst/ bin.includes = META-INF/,\ .,\ plugin.properties diff --git a/org.eclipse.jgit.gpg.bc.test/pom.xml b/org.eclipse.jgit.gpg.bc.test/pom.xml index fd477b463..aa09d3f00 100644 --- a/org.eclipse.jgit.gpg.bc.test/pom.xml +++ b/org.eclipse.jgit.gpg.bc.test/pom.xml @@ -85,6 +85,12 @@ src/ tst/ + + + tst-rsrc/ + + + org.apache.maven.plugins diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A.asc new file mode 100644 index 000000000..355462c09 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A.key b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A.key new file mode 100644 index 000000000..afa459c83 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A.key differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9.asc new file mode 100644 index 000000000..362e2108e Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9.key b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9.key new file mode 100644 index 000000000..cef72f623 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9.key differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11.asc new file mode 100644 index 000000000..f41201990 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11.key b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11.key new file mode 100644 index 000000000..b8765aaac Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11.key differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/F727FAB884DA3BD402B6E0F5472E108D21033124.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/F727FAB884DA3BD402B6E0F5472E108D21033124.asc new file mode 100644 index 000000000..c6e0408b3 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/F727FAB884DA3BD402B6E0F5472E108D21033124.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/F727FAB884DA3BD402B6E0F5472E108D21033124.key b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/F727FAB884DA3BD402B6E0F5472E108D21033124.key new file mode 100644 index 000000000..63617c09b Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/F727FAB884DA3BD402B6E0F5472E108D21033124.key differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool256.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool256.asc new file mode 100644 index 000000000..8427cfcc0 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool256.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool384.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool384.asc new file mode 100644 index 000000000..bdb20fe93 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool384.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool512.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool512.asc new file mode 100644 index 000000000..5b4bca2c6 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/brainpool512.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/dsa-elgamal.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/dsa-elgamal.asc new file mode 100644 index 000000000..db067325c Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/dsa-elgamal.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/ed25519.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/ed25519.asc new file mode 100644 index 000000000..636a5a95f Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/ed25519.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/faked.key b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/faked.key new file mode 100644 index 000000000..405afad42 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/faked.key differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp256.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp256.asc new file mode 100644 index 000000000..fd1509e9b Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp256.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp384.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp384.asc new file mode 100644 index 000000000..b2b59959e Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp384.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp521.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp521.asc new file mode 100644 index 000000000..db18f81a2 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/nistp521.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/rsa.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/rsa.asc new file mode 100644 index 000000000..e74df7a2a Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/rsa.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/secp256k1.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/secp256k1.asc new file mode 100644 index 000000000..837f8a867 Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/secp256k1.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/x25519.asc b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/x25519.asc new file mode 100644 index 000000000..d531d7a8b Binary files /dev/null and b/org.eclipse.jgit.gpg.bc.test/tst-rsrc/org/eclipse/jgit/gpg/bc/internal/keys/x25519.asc differ diff --git a/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocatorTest.java b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocatorTest.java index 744620163..5f4337870 100644 --- a/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocatorTest.java +++ b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocatorTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019, Thomas Wolf and others + * Copyright (C) 2019, 2020 Thomas Wolf and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -53,7 +53,7 @@ public void testFullEmail() throws Exception { assertFalse(match(USER_ID, "")); assertFalse(match(USER_ID, "")); assertFalse(match(USER_ID, "")); - assertFalse(match(USER_ID, "")); + assertTrue(match(USER_ID, "")); assertFalse(match(USER_ID.substring(0, USER_ID.length() - 1), "")); assertFalse(match("", "<>")); @@ -72,8 +72,8 @@ public void testPartialEmail() throws Exception { assertFalse(match(USER_ID, "@ ")); assertFalse(match(USER_ID, "@")); assertFalse(match(USER_ID, "@Heine")); - assertFalse(match(USER_ID, "@HeinrichH")); - assertFalse(match(USER_ID, "@Heinrich")); + assertTrue(match(USER_ID, "@HeinrichH")); + assertTrue(match(USER_ID, "@Heinrich")); assertFalse(match("", "@")); assertFalse(match("", "@h")); } @@ -110,6 +110,7 @@ public void testSubstringAsterisk() throws Exception { public void testExplicitFingerprint() throws Exception { assertFalse(match("John Fade ", "0xfade")); assertFalse(match("John Fade <0xfade@example.com>", "0xfade")); + assertFalse(match("John Fade <0xfade@example.com>", "0xFADE")); assertFalse(match("", "0xfade")); } @@ -128,7 +129,7 @@ public void testZeroX() throws Exception { assertTrue(match("John Fade <0xfade@example.com>", "*0xfade")); assertTrue(match("John Fade <0xfade@example.com>", "*0xFADE")); assertTrue(match("John Fade <0xfade@example.com>", "@0xfade")); - assertFalse(match("John Fade <0xfade@example.com>", "@0xFADE")); + assertTrue(match("John Fade <0xfade@example.com>", "@0xFADE")); assertFalse(match("", "0x")); } } diff --git a/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/KeyGrip25519Test.java b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/KeyGrip25519Test.java new file mode 100644 index 000000000..e30080258 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/KeyGrip25519Test.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import static org.junit.Assert.assertEquals; + +import java.math.BigInteger; +import java.util.Locale; + +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.util.encoders.Hex; +import org.eclipse.jgit.util.sha1.SHA1; +import org.junit.Test; + +public class KeyGrip25519Test { + + interface Hash { + byte[] hash(SHA1 sha, BigInteger q) throws PGPException; + } + + private void assertKeyGrip(String key, String expectedKeyGrip, Hash hash) + throws Exception { + SHA1 grip = SHA1.newInstance(); + grip.setDetectCollision(false); + BigInteger pk = new BigInteger(key, 16); + byte[] keyGrip = hash.hash(grip, pk); + assertEquals("Keygrip should match", expectedKeyGrip, + Hex.toHexString(keyGrip).toUpperCase(Locale.ROOT)); + } + + @Test + public void testCompressed() throws Exception { + assertKeyGrip("40" + + "773E72848C1FD5F9652B29E2E7AF79571A04990E96F2016BF4E0EC1890C2B7DB", + "9DB6C64A38830F4960701789475520BE8C821F47", + KeyGrip::hashEd25519); + } + + @Test + public void testCompressedNoPrefix() throws Exception { + assertKeyGrip( + "773E72848C1FD5F9652B29E2E7AF79571A04990E96F2016BF4E0EC1890C2B7DB", + "9DB6C64A38830F4960701789475520BE8C821F47", + KeyGrip::hashEd25519); + } + + @Test + public void testCurve25519() throws Exception { + assertKeyGrip("40" + + "918C1733127F6BF2646FAE3D081A18AE77111C903B906310B077505EFFF12740", + "0F89A565D3EA187CE839332398F5D480677DF49C", + KeyGrip::hashCurve25519); + } +} diff --git a/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/KeyGripTest.java b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/KeyGripTest.java new file mode 100644 index 000000000..a4aaf40d9 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/KeyGripTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.security.Security; +import java.util.Iterator; +import java.util.Locale; +import java.util.function.Consumer; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPPublicKeyRing; +import org.bouncycastle.openpgp.PGPPublicKeyRingCollection; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.util.encoders.Hex; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class KeyGripTest { + + @BeforeClass + public static void ensureBC() { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + protected static class TestData { + + String filename; + + String[] expectedKeyGrips; + + TestData(String filename, String... keyGrips) { + this.filename = filename; + this.expectedKeyGrips = keyGrips; + } + + @Override + public String toString() { + return filename; + } + } + + @Parameters(name = "{0}") + public static TestData[] initTestData() { + return new TestData[] { + new TestData("rsa.asc", + "D148210FAF36468055B83D0F5A6DEB83FBC8E864", + "A5E4CD2CBBE44A16E4D6EC05C2E3C3A599DC763C"), + new TestData("dsa-elgamal.asc", + "552286BEB2999F0A9E26A50385B90D9724001187", + "CED7034A8EB5F4CE90DF99147EC33D86FCD3296C"), + new TestData("brainpool256.asc", + "A01BAA22A72F09A0FF0A1D4CBCE70844DD52DDD7", + "C1678B7DE5F144C93B89468D5F9764ACE182ED36"), + new TestData("brainpool384.asc", + "2F25DB025DEBF3EA2715350209B985829B04F50A", + "B6BD8B81F75AF914163D97DF8DE8F6FC64C283F8"), + new TestData("brainpool512.asc", + "5A484F56AB4B8B6583B6365034999F6543FAE1AE", + "9133E4A7E8FC8515518DF444C3F2F247EEBBADEC"), + new TestData("nistp256.asc", + "FC81AECE90BCE6E54D0D637D266109783AC8DAC0", + "A56DC8DB8355747A809037459B4258B8A743EAB5"), + new TestData("nistp384.asc", + "A1338230AED1C9C125663518470B49056C9D1733", + "797A83FE041FFE06A7F4B1D32C6F4AE0F6D87ADF"), + new TestData("nistp521.asc", + "D91B789603EC9138AA20342A2B6DC86C81B70F5D", + "FD048B2CA1919CB241DC8A2C7FA3E742EF343DCA"), + new TestData("secp256k1.asc", + "498B89C485489BA16B40755C0EBA580166393074", + "48FFED40D018747363BDEFFDD404D1F4870F8064"), + new TestData("ed25519.asc", + "940D97D75C306D737A59A98EAFF1272832CEDC0B"), + new TestData("x25519.asc", + "A77DC8173DA6BEE126F5BD6F5A14E01200B52FCE", + "636C983EDB558527BA82780B52CB5DAE011BE46B") + }; + } + + // Injected by JUnit + @Parameter + public TestData data; + + private void readAsc(InputStream in, Consumer process) + throws IOException, PGPException { + PGPPublicKeyRingCollection pgpPub = new PGPPublicKeyRingCollection( + PGPUtil.getDecoderStream(in), new JcaKeyFingerprintCalculator()); + + Iterator keyRings = pgpPub.getKeyRings(); + while (keyRings.hasNext()) { + PGPPublicKeyRing keyRing = keyRings.next(); + + Iterator keys = keyRing.getPublicKeys(); + while (keys.hasNext()) { + process.accept(keys.next()); + } + } + } + + @Test + public void testGrip() throws Exception { + try (InputStream in = this.getClass() + .getResourceAsStream(data.filename)) { + int index[] = { 0 }; + readAsc(in, key -> { + byte[] keyGrip = null; + try { + keyGrip = KeyGrip.getKeyGrip(key); + } catch (PGPException e) { + throw new RuntimeException(e); + } + assertTrue("More keys than expected", + index[0] < data.expectedKeyGrips.length); + assertEquals("Wrong keygrip", data.expectedKeyGrips[index[0]++], + Hex.toHexString(keyGrip).toUpperCase(Locale.ROOT)); + }); + assertEquals("Missing keys", data.expectedKeyGrips.length, + index[0]); + } + } +} diff --git a/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/SecretKeysTest.java b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/SecretKeysTest.java new file mode 100644 index 000000000..5e5e30331 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc.test/tst/org/eclipse/jgit/gpg/bc/internal/keys/SecretKeysTest.java @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.Security; +import java.util.Iterator; + +import javax.crypto.Cipher; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPPublicKeyRing; +import org.bouncycastle.openpgp.PGPPublicKeyRingCollection; +import org.bouncycastle.openpgp.PGPSecretKey; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.PGPDigestCalculatorProvider; +import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class SecretKeysTest { + + @BeforeClass + public static void ensureBC() { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + private static volatile Boolean haveOCB; + + private static boolean ocbAvailable() { + Boolean haveIt = haveOCB; + if (haveIt != null) { + return haveIt.booleanValue(); + } + try { + Cipher c = Cipher.getInstance("AES/OCB/NoPadding"); //$NON-NLS-1$ + if (c == null) { + haveOCB = Boolean.FALSE; + return false; + } + } catch (NoClassDefFoundError | Exception e) { + haveOCB = Boolean.FALSE; + return false; + } + haveOCB = Boolean.TRUE; + return true; + } + + private static class TestData { + + final String name; + + final boolean encrypted; + + final boolean keyValue; + + TestData(String name, boolean encrypted, boolean keyValue) { + this.name = name; + this.encrypted = encrypted; + this.keyValue = keyValue; + } + + @Override + public String toString() { + return name; + } + } + + @Parameters(name = "{0}") + public static TestData[] initTestData() { + return new TestData[] { + new TestData("AFDA8EA10E185ACF8C0D0F8885A0EF61A72ECB11", false, false), + new TestData("2FB05DBB70FC07CB84C13431F640CA6CEA1DBF8A", false, true), + new TestData("66CCECEC2AB46A9735B10FEC54EDF9FD0F77BAF9", true, true), + new TestData("F727FAB884DA3BD402B6E0F5472E108D21033124", true, true), + new TestData("faked", false, true) }; + } + + private static byte[] readTestKey(String filename) throws Exception { + try (InputStream in = new BufferedInputStream( + SecretKeysTest.class.getResourceAsStream(filename))) { + return SecretKeys.keyFromNameValueFormat(in); + } + } + + private static PGPPublicKey readAsc(InputStream in) + throws IOException, PGPException { + PGPPublicKeyRingCollection pgpPub = new PGPPublicKeyRingCollection( + PGPUtil.getDecoderStream(in), new JcaKeyFingerprintCalculator()); + + Iterator keyRings = pgpPub.getKeyRings(); + while (keyRings.hasNext()) { + PGPPublicKeyRing keyRing = keyRings.next(); + + Iterator keys = keyRing.getPublicKeys(); + if (keys.hasNext()) { + return keys.next(); + } + } + return null; + } + + // Injected by JUnit + @Parameter + public TestData data; + + @Test + public void testKeyRead() throws Exception { + if (data.keyValue) { + byte[] bytes = readTestKey(data.name + ".key"); + assertEquals('(', bytes[0]); + assertEquals(')', bytes[bytes.length - 1]); + } + try (InputStream pubIn = this.getClass() + .getResourceAsStream(data.name + ".asc")) { + if (pubIn != null) { + PGPPublicKey publicKey = readAsc(pubIn); + // Do a full test trying to load the secret key. + PGPDigestCalculatorProvider calculatorProvider = new JcaPGPDigestCalculatorProviderBuilder() + .build(); + try (InputStream in = new BufferedInputStream(this.getClass() + .getResourceAsStream(data.name + ".key"))) { + PGPSecretKey secretKey = SecretKeys.readSecretKey(in, + calculatorProvider, + data.encrypted ? () -> "nonsense".toCharArray() + : null, + publicKey); + assertNotNull(secretKey); + } catch (PGPException e) { + // Currently we may not be able to load OCB-encrypted keys. + assertTrue(e.getMessage().contains("OCB")); + assertTrue(data.encrypted); + assertFalse(ocbAvailable()); + } + } + } + } + +} diff --git a/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF b/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF index b59a850ea..48aad320d 100644 --- a/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF @@ -8,22 +8,30 @@ Bundle-Vendor: %Bundle-Vendor Bundle-Localization: plugin Bundle-Version: 6.0.0.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.bouncycastle.bcpg;version="[1.65.0,2.0.0)", +Import-Package: org.bouncycastle.asn1;version="[1.65.0,2.0.0)", + org.bouncycastle.asn1.cryptlib;version="[1.65.0,2.0.0)", + org.bouncycastle.asn1.x9;version="[1.65.0,2.0.0)", + org.bouncycastle.bcpg;version="[1.65.0,2.0.0)", + org.bouncycastle.bcpg.sig;version="[1.65.0,2.0.0)", + org.bouncycastle.crypto.ec;version="[1.65.0,2.0.0)", org.bouncycastle.gpg;version="[1.65.0,2.0.0)", org.bouncycastle.gpg.keybox;version="[1.65.0,2.0.0)", org.bouncycastle.gpg.keybox.jcajce;version="[1.65.0,2.0.0)", + org.bouncycastle.jcajce.interfaces;version="[1.65.0,2.0.0)", + org.bouncycastle.jcajce.util;version="[1.65.0,2.0.0)", org.bouncycastle.jce.provider;version="[1.65.0,2.0.0)", + org.bouncycastle.math.ec;version="[1.65.0,2.0.0)", + org.bouncycastle.math.field;version="[1.65.0,2.0.0)", org.bouncycastle.openpgp;version="[1.65.0,2.0.0)", + org.bouncycastle.openpgp.jcajce;version="[1.65.0,2.0.0)", org.bouncycastle.openpgp.operator;version="[1.65.0,2.0.0)", org.bouncycastle.openpgp.operator.jcajce;version="[1.65.0,2.0.0)", + org.bouncycastle.util;version="[1.65.0,2.0.0)", org.bouncycastle.util.encoders;version="[1.65.0,2.0.0)", + org.bouncycastle.util.io;version="[1.65.0,2.0.0)", org.eclipse.jgit.annotations;version="[6.0.0,6.1.0)", org.eclipse.jgit.api.errors;version="[6.0.0,6.1.0)", - org.eclipse.jgit.errors;version="[6.0.0,6.1.0)", - org.eclipse.jgit.lib;version="[6.0.0,6.1.0)", - org.eclipse.jgit.nls;version="[6.0.0,6.1.0)", - org.eclipse.jgit.transport;version="[6.0.0,6.1.0)", - org.eclipse.jgit.util;version="[6.0.0,6.1.0)", org.slf4j;version="[1.7.0,2.0.0)" -Export-Package: org.eclipse.jgit.gpg.bc.internal;version="6.0.0"; - x-friends:="org.eclipse.jgit.gpg.bc.test" +Export-Package: org.eclipse.jgit.gpg.bc;version="6.0.0", + org.eclipse.jgit.gpg.bc.internal;version="6.0.0";x-friends:="org.eclipse.jgit.gpg.bc.test", + org.eclipse.jgit.gpg.bc.internal.keys;version="6.0.0";x-friends:="org.eclipse.jgit.gpg.bc.test" diff --git a/org.eclipse.jgit.gpg.bc/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit.gpg.bc/META-INF/SOURCE-MANIFEST.MF index 9e0d921ea..58bd8ea04 100644 --- a/org.eclipse.jgit.gpg.bc/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit.gpg.bc/META-INF/SOURCE-MANIFEST.MF @@ -5,3 +5,4 @@ Bundle-SymbolicName: org.eclipse.jgit.gpg.bc.source Bundle-Vendor: Eclipse.org - JGit Bundle-Version: 6.0.0.qualifier Eclipse-SourceBundle: org.eclipse.jgit.gpg.bc;version="6.0.0.qualifier";roots="." + diff --git a/org.eclipse.jgit.gpg.bc/about.html b/org.eclipse.jgit.gpg.bc/about.html index f971af18d..fc527d5a3 100644 --- a/org.eclipse.jgit.gpg.bc/about.html +++ b/org.eclipse.jgit.gpg.bc/about.html @@ -11,7 +11,7 @@ margin: 0.25in 0.5in 0.25in 0.5in; tab-interval: 0.5in; } - p { + p { margin-left: auto; margin-top: 0.5em; margin-bottom: 0.5em; @@ -36,60 +36,53 @@

Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.

All rights reserved.

-

Redistribution and use in source and binary forms, with or without modification, +

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -

  • Redistributions of source code must retain the above copyright notice, +
    • Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
      • -
    • Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation +
    • Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
      • -
    • Neither the name of the Eclipse Foundation, Inc. nor the names of its - contributors may be used to endorse or promote products derived from +
    • Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this software without specific prior written permission.

    -

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

    -

    SHA-1 UbcCheck - MIT

    +

    org.eclipse.jgit.gpg.bc.internal.keys.SExprParser - MIT

    -

    Copyright (c) 2017:

    -
    -Marc Stevens -Cryptology Group -Centrum Wiskunde & Informatica -P.O. Box 94079, 1090 GB Amsterdam, Netherlands -marc@marc-stevens.nl -
    -
    -Dan Shumow -Microsoft Research -danshu@microsoft.com -
    -

    Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +

    Copyright (c) 2000-2021 The Legion of the Bouncy Castle Inc. +(https://www.bouncycastle.org)

    + +

    +Permission is hereby granted, free of charge, to any person obtaining a copy of this software +and associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: +

    +

    +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. +

    +

    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE.

    -
    • The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software.
    -

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE.

    diff --git a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory new file mode 100644 index 000000000..17ab30fba --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory @@ -0,0 +1 @@ +org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSignatureVerifierFactory \ No newline at end of file diff --git a/org.eclipse.jgit.gpg.bc/resources/org/eclipse/jgit/gpg/bc/internal/BCText.properties b/org.eclipse.jgit.gpg.bc/resources/org/eclipse/jgit/gpg/bc/internal/BCText.properties index 1441c63e8..e4b1baba1 100644 --- a/org.eclipse.jgit.gpg.bc/resources/org/eclipse/jgit/gpg/bc/internal/BCText.properties +++ b/org.eclipse.jgit.gpg.bc/resources/org/eclipse/jgit/gpg/bc/internal/BCText.properties @@ -1,11 +1,36 @@ +corrupt25519Key=Ed25519/Curve25519 public key has wrong length: {0} credentialPassphrase=Passphrase -gpgFailedToParseSecretKey=Failed to parse secret key file in directory: {0}. Is the entered passphrase correct? +cryptCipherError=Cannot create cipher to decrypt: {0} +cryptWrongDecryptedLength=Decrypted key has wrong length; expected {0} bytes, got only {1} bytes +gpgFailedToParseSecretKey=Failed to parse secret key file {0}. Is the entered passphrase correct? gpgNoCredentialsProvider=missing credentials provider +gpgNoKeygrip=Cannot find key {0}: cannot determine key grip gpgNoKeyring=neither pubring.kbx nor secring.gpg files found gpgNoKeyInLegacySecring=no matching secret key found in legacy secring.gpg for key or user id: {0} gpgNoPublicKeyFound=Unable to find a public-key with key or user id: {0} gpgNoSecretKeyForPublicKey=unable to find associated secret key for public key: {0} +gpgNoSuchAlgorithm=Cannot decrypt encrypted secret key: encryption algorithm {0} is not available gpgNotASigningKey=Secret key ({0}) is not suitable for signing gpgKeyInfo=GPG Key (fingerprint {0}) gpgSigningCancelled=Signing was cancelled +nonSignatureError=Signature does not decode into a signature object +secretKeyTooShort=Secret key file corrupt; only {0} bytes read +sexprHexNotClosed=Hex number in s-expression not closed +sexprHexOdd=Hex number in s-expression has an odd number of digits +sexprStringInvalidEscape=Invalid escape {0} in s-expression +sexprStringInvalidEscapeAtEnd=Invalid s-expression: quoted string ends with escape character +sexprStringInvalidHexEscape=Invalid hex escape in s-expression +sexprStringInvalidOctalEscape=Invalid octal escape in s-expression +sexprStringNotClosed=String in s-expression not closed +sexprUnhandled=Unhandled token {0} in s-expression +signatureInconsistent=Inconsistent signature; key ID {0} does not match issuer fingerprint {1} +signatureKeyLookupError=Error occurred while looking for public key +signatureNoKeyInfo=No way to determine a public key from the signature +signatureNoPublicKey=No public key found to verify the signature +signatureParseError=Signature cannot be parsed +signatureVerificationError=Signature verification failed unableToSignCommitNoSecretKey=Unable to sign commit. Signing key not available. +uncompressed25519Key=Cannot handle ed25519 public key with uncompressed data: {0} +unknownCurve=Unknown curve {0} +unknownCurveParameters=Curve {0} does not have a prime field +unknownKeyType=Unknown key type {0} \ No newline at end of file diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java new file mode 100644 index 000000000..fdd1a2b11 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc; + +import org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSigner; +import org.eclipse.jgit.lib.GpgSigner; + +/** + * Factory for creating a {@link GpgSigner} based on Bouncy Castle. + * + * @since 5.11 + */ +public final class BouncyCastleGpgSignerFactory { + + private BouncyCastleGpgSignerFactory() { + // No instantiation + } + + /** + * Creates a new {@link GpgSigner}. + * + * @return the {@link GpgSigner} + */ + public static GpgSigner create() { + return new BouncyCastleGpgSigner(); + } +} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BCText.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BCText.java index 1a00b0fc7..aedf8a5be 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BCText.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BCText.java @@ -1,3 +1,12 @@ +/* + * Copyright (C) 2018, 2021 Salesforce and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ package org.eclipse.jgit.gpg.bc.internal; import org.eclipse.jgit.nls.NLS; @@ -18,16 +27,41 @@ public static BCText get() { } // @formatter:off + /***/ public String corrupt25519Key; /***/ public String credentialPassphrase; + /***/ public String cryptCipherError; + /***/ public String cryptWrongDecryptedLength; /***/ public String gpgFailedToParseSecretKey; /***/ public String gpgNoCredentialsProvider; + /***/ public String gpgNoKeygrip; /***/ public String gpgNoKeyring; /***/ public String gpgNoKeyInLegacySecring; /***/ public String gpgNoPublicKeyFound; /***/ public String gpgNoSecretKeyForPublicKey; + /***/ public String gpgNoSuchAlgorithm; /***/ public String gpgNotASigningKey; /***/ public String gpgKeyInfo; /***/ public String gpgSigningCancelled; + /***/ public String nonSignatureError; + /***/ public String secretKeyTooShort; + /***/ public String sexprHexNotClosed; + /***/ public String sexprHexOdd; + /***/ public String sexprStringInvalidEscape; + /***/ public String sexprStringInvalidEscapeAtEnd; + /***/ public String sexprStringInvalidHexEscape; + /***/ public String sexprStringInvalidOctalEscape; + /***/ public String sexprStringNotClosed; + /***/ public String sexprUnhandled; + /***/ public String signatureInconsistent; + /***/ public String signatureKeyLookupError; + /***/ public String signatureNoKeyInfo; + /***/ public String signatureNoPublicKey; + /***/ public String signatureParseError; + /***/ public String signatureVerificationError; /***/ public String unableToSignCommitNoSecretKey; + /***/ public String uncompressed25519Key; + /***/ public String unknownCurve; + /***/ public String unknownCurveParameters; + /***/ public String unknownKeyType; } diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocator.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocator.java index eca45072b..cf4d3d234 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocator.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyLocator.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, 2020 Salesforce and others + * Copyright (C) 2018, 2021 Salesforce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -14,25 +14,22 @@ import java.io.BufferedInputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.InvalidPathException; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; import java.text.MessageFormat; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.Locale; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.bouncycastle.gpg.SExprParser; import org.bouncycastle.gpg.keybox.BlobType; import org.bouncycastle.gpg.keybox.KeyBlob; import org.bouncycastle.gpg.keybox.KeyBox; @@ -50,15 +47,15 @@ import org.bouncycastle.openpgp.PGPSecretKeyRingCollection; import org.bouncycastle.openpgp.PGPSignature; import org.bouncycastle.openpgp.PGPUtil; -import org.bouncycastle.openpgp.operator.PBEProtectionRemoverFactory; import org.bouncycastle.openpgp.operator.PGPDigestCalculatorProvider; import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; -import org.bouncycastle.openpgp.operator.jcajce.JcePBEProtectionRemoverFactory; import org.bouncycastle.util.encoders.Hex; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.errors.UnsupportedCredentialItem; +import org.eclipse.jgit.gpg.bc.internal.keys.KeyGrip; +import org.eclipse.jgit.gpg.bc.internal.keys.SecretKeys; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; @@ -78,17 +75,10 @@ private static class NoOpenPgpKeyException extends Exception { } - /** Thrown if we try to read an encrypted private key without password. */ - private static class EncryptedPgpKeyException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - } - private static final Logger log = LoggerFactory .getLogger(BouncyCastleGpgKeyLocator.class); - private static final Path GPG_DIRECTORY = findGpgDirectory(); + static final Path GPG_DIRECTORY = findGpgDirectory(); private static final Path USER_KEYBOX_PATH = GPG_DIRECTORY .resolve("pubring.kbx"); //$NON-NLS-1$ @@ -155,16 +145,13 @@ public BouncyCastleGpgKeyLocator(String signingKey, private PGPSecretKey attemptParseSecretKey(Path keyFile, PGPDigestCalculatorProvider calculatorProvider, - PBEProtectionRemoverFactory passphraseProvider, - PGPPublicKey publicKey) { + SecretKeys.PassphraseSupplier passphraseSupplier, + PGPPublicKey publicKey) + throws IOException, PGPException, CanceledException, + UnsupportedCredentialItem, URISyntaxException { try (InputStream in = newInputStream(keyFile)) { - return new SExprParser(calculatorProvider).parseSecretKey( - new BufferedInputStream(in), passphraseProvider, publicKey); - } catch (IOException | PGPException | ClassCastException e) { - if (log.isDebugEnabled()) - log.debug("Ignoring unreadable file '{}': {}", keyFile, //$NON-NLS-1$ - e.getMessage(), e); - return null; + return SecretKeys.readSecretKey(in, calculatorProvider, + passphraseSupplier, publicKey); } } @@ -219,33 +206,60 @@ static boolean containsSigningKey(String userId, String signingKeySpec) { int stop = toMatch.indexOf('>'); return begin >= 0 && end > begin + 1 && stop > 0 && userId.substring(begin + 1, end) - .equals(toMatch.substring(0, stop)); + .equalsIgnoreCase(toMatch.substring(0, stop)); } case '@': { int begin = userId.indexOf('<'); int end = userId.indexOf('>', begin + 1); return begin >= 0 && end > begin + 1 - && userId.substring(begin + 1, end).contains(toMatch); + && containsIgnoreCase(userId.substring(begin + 1, end), + toMatch); } default: if (toMatch.trim().isEmpty()) { return false; } - return userId.toLowerCase(Locale.ROOT) - .contains(toMatch.toLowerCase(Locale.ROOT)); + return containsIgnoreCase(userId, toMatch); } } - private String toFingerprint(String keyId) { + private static boolean containsIgnoreCase(String a, String b) { + int alength = a.length(); + int blength = b.length(); + for (int i = 0; i + blength <= alength; i++) { + if (a.regionMatches(true, i, b, 0, blength)) { + return true; + } + } + return false; + } + + private static String toFingerprint(String keyId) { if (keyId.startsWith("0x")) { //$NON-NLS-1$ return keyId.substring(2); } return keyId; } - private PGPPublicKey findPublicKeyByKeyId(KeyBlob keyBlob) + static PGPPublicKey findPublicKey(String fingerprint, String keySpec) + throws IOException, PGPException { + PGPPublicKey result = findPublicKeyInPubring(USER_PGP_PUBRING_FILE, + fingerprint, keySpec); + if (result == null && exists(USER_KEYBOX_PATH)) { + try { + result = findPublicKeyInKeyBox(USER_KEYBOX_PATH, fingerprint, + keySpec); + } catch (NoSuchAlgorithmException | NoSuchProviderException + | IOException | NoOpenPgpKeyException e) { + log.error(e.getMessage(), e); + } + } + return result; + } + + private static PGPPublicKey findPublicKeyByKeyId(KeyBlob keyBlob, + String keyId) throws IOException { - String keyId = toFingerprint(signingKey).toLowerCase(Locale.ROOT); if (keyId.isEmpty()) { return null; } @@ -259,10 +273,11 @@ private PGPPublicKey findPublicKeyByKeyId(KeyBlob keyBlob) return null; } - private PGPPublicKey findPublicKeyByUserId(KeyBlob keyBlob) + private static PGPPublicKey findPublicKeyByUserId(KeyBlob keyBlob, + String keySpec) throws IOException { for (UserID userID : keyBlob.getUserIds()) { - if (containsSigningKey(userID.getUserIDAsString(), signingKey)) { + if (containsSigningKey(userID.getUserIDAsString(), keySpec)) { return getSigningPublicKey(keyBlob); } } @@ -274,6 +289,10 @@ private PGPPublicKey findPublicKeyByUserId(KeyBlob keyBlob) * * @param keyboxFile * the KeyBox file + * @param keyId + * to look for, may be null + * @param keySpec + * to look for * @return publicKey the public key (maybe null) * @throws IOException * in case of problems reading the file @@ -282,19 +301,22 @@ private PGPPublicKey findPublicKeyByUserId(KeyBlob keyBlob) * @throws NoOpenPgpKeyException * if the file does not contain any OpenPGP key */ - private PGPPublicKey findPublicKeyInKeyBox(Path keyboxFile) + private static PGPPublicKey findPublicKeyInKeyBox(Path keyboxFile, + String keyId, String keySpec) throws IOException, NoSuchAlgorithmException, NoSuchProviderException, NoOpenPgpKeyException { KeyBox keyBox = readKeyBoxFile(keyboxFile); + String id = keyId != null ? keyId + : toFingerprint(keySpec).toLowerCase(Locale.ROOT); boolean hasOpenPgpKey = false; for (KeyBlob keyBlob : keyBox.getKeyBlobs()) { if (keyBlob.getType() == BlobType.OPEN_PGP_BLOB) { hasOpenPgpKey = true; - PGPPublicKey key = findPublicKeyByKeyId(keyBlob); + PGPPublicKey key = findPublicKeyByKeyId(keyBlob, id); if (key != null) { return key; } - key = findPublicKeyByUserId(keyBlob); + key = findPublicKeyByUserId(keyBlob, keySpec); if (key != null) { return key; } @@ -338,7 +360,8 @@ public BouncyCastleGpgKey findSecretKey() throws IOException, // pubring.gpg also try secring.gpg to find the secret key. if (exists(USER_KEYBOX_PATH)) { try { - publicKey = findPublicKeyInKeyBox(USER_KEYBOX_PATH); + publicKey = findPublicKeyInKeyBox(USER_KEYBOX_PATH, null, + signingKey); if (publicKey != null) { key = findSecretKeyForKeyBoxPublicKey(publicKey, USER_KEYBOX_PATH); @@ -361,7 +384,8 @@ public BouncyCastleGpgKey findSecretKey() throws IOException, } } if (exists(USER_PGP_PUBRING_FILE)) { - publicKey = findPublicKeyInPubring(USER_PGP_PUBRING_FILE); + publicKey = findPublicKeyInPubring(USER_PGP_PUBRING_FILE, null, + signingKey); if (publicKey != null) { // GPG < 2.1 may have both; the agent using the directory // and gpg using secring.gpg. GPG >= 2.1 delegates all @@ -433,67 +457,59 @@ private BouncyCastleGpgKey findSecretKeyForKeyBoxPublicKey( PGPPublicKey publicKey, Path userKeyboxPath) throws PGPException, CanceledException, UnsupportedCredentialItem, URISyntaxException { - /* - * this is somewhat brute-force but there doesn't seem to be another - * way; we have to walk all private key files we find and try to open - * them - */ - - PGPDigestCalculatorProvider calculatorProvider = new JcaPGPDigestCalculatorProviderBuilder() - .build(); - - try (Stream keyFiles = Files.walk(USER_SECRET_KEY_DIR)) { - List allPaths = keyFiles.filter(Files::isRegularFile) - .collect(Collectors.toCollection(ArrayList::new)); - if (allPaths.isEmpty()) { - return null; + byte[] keyGrip = null; + try { + keyGrip = KeyGrip.getKeyGrip(publicKey); + } catch (PGPException e) { + throw new PGPException( + MessageFormat.format(BCText.get().gpgNoKeygrip, + Hex.toHexString(publicKey.getFingerprint())), + e); + } + String filename = Hex.toHexString(keyGrip).toUpperCase(Locale.ROOT) + + ".key"; //$NON-NLS-1$ + Path keyFile = USER_SECRET_KEY_DIR.resolve(filename); + if (!Files.exists(keyFile)) { + return null; + } + boolean clearPrompt = false; + try { + PGPDigestCalculatorProvider calculatorProvider = new JcaPGPDigestCalculatorProviderBuilder() + .build(); + clearPrompt = true; + PGPSecretKey secretKey = null; + try { + secretKey = attemptParseSecretKey(keyFile, calculatorProvider, + () -> passphrasePrompt.getPassphrase( + publicKey.getFingerprint(), userKeyboxPath), + publicKey); + } catch (PGPException e) { + throw new PGPException(MessageFormat.format( + BCText.get().gpgFailedToParseSecretKey, + keyFile.toAbsolutePath()), e); } - PBEProtectionRemoverFactory passphraseProvider = p -> { - throw new EncryptedPgpKeyException(); - }; - for (int attempts = 0; attempts < 2; attempts++) { - // Second pass will traverse only the encrypted keys with a real - // passphrase provider. - Iterator pathIterator = allPaths.iterator(); - while (pathIterator.hasNext()) { - Path keyFile = pathIterator.next(); - try { - PGPSecretKey secretKey = attemptParseSecretKey(keyFile, - calculatorProvider, passphraseProvider, - publicKey); - pathIterator.remove(); - if (secretKey != null) { - if (!secretKey.isSigningKey()) { - throw new PGPException(MessageFormat.format( - BCText.get().gpgNotASigningKey, - signingKey)); - } - return new BouncyCastleGpgKey(secretKey, - userKeyboxPath); - } - } catch (EncryptedPgpKeyException e) { - // Ignore; we'll try again. - } + if (secretKey != null) { + if (!secretKey.isSigningKey()) { + throw new PGPException(MessageFormat.format( + BCText.get().gpgNotASigningKey, signingKey)); } - if (attempts > 0 || allPaths.isEmpty()) { - break; - } - // allPaths contains only the encrypted keys now. - passphraseProvider = new JcePBEProtectionRemoverFactory( - passphrasePrompt.getPassphrase( - publicKey.getFingerprint(), userKeyboxPath)); + clearPrompt = false; + return new BouncyCastleGpgKey(secretKey, userKeyboxPath); } - - passphrasePrompt.clear(); return null; } catch (RuntimeException e) { - passphrasePrompt.clear(); throw e; + } catch (FileNotFoundException | NoSuchFileException e) { + clearPrompt = false; + return null; } catch (IOException e) { - passphrasePrompt.clear(); throw new PGPException(MessageFormat.format( BCText.get().gpgFailedToParseSecretKey, - USER_SECRET_KEY_DIR.toAbsolutePath()), e); + keyFile.toAbsolutePath()), e); + } finally { + if (clearPrompt) { + passphrasePrompt.clear(); + } } } @@ -551,6 +567,11 @@ private PGPSecretKey findSecretKeyInLegacySecring(String signingkey, * Return the first public key matching the key id ({@link #signingKey}. * * @param pubringFile + * to search + * @param keyId + * to look for, may be null + * @param keySpec + * to look for * * @return the PGP public key, or {@code null} if none found * @throws IOException @@ -558,14 +579,16 @@ private PGPSecretKey findSecretKeyInLegacySecring(String signingkey, * @throws PGPException * on BouncyCastle errors */ - private PGPPublicKey findPublicKeyInPubring(Path pubringFile) + private static PGPPublicKey findPublicKeyInPubring(Path pubringFile, + String keyId, String keySpec) throws IOException, PGPException { try (InputStream in = newInputStream(pubringFile)) { PGPPublicKeyRingCollection pgpPub = new PGPPublicKeyRingCollection( new BufferedInputStream(in), new JcaKeyFingerprintCalculator()); - String keyId = toFingerprint(signingKey).toLowerCase(Locale.ROOT); + String id = keyId != null ? keyId + : toFingerprint(keySpec).toLowerCase(Locale.ROOT); Iterator keyrings = pgpPub.getKeyRings(); while (keyrings.hasNext()) { PGPPublicKeyRing keyRing = keyrings.next(); @@ -575,30 +598,33 @@ private PGPPublicKey findPublicKeyInPubring(Path pubringFile) // try key id String fingerprint = Hex.toHexString(key.getFingerprint()) .toLowerCase(Locale.ROOT); - if (fingerprint.endsWith(keyId)) { + if (fingerprint.endsWith(id)) { return key; } // try user id Iterator userIDs = key.getUserIDs(); while (userIDs.hasNext()) { String userId = userIDs.next(); - if (containsSigningKey(userId, signingKey)) { + if (containsSigningKey(userId, keySpec)) { return key; } } } } + } catch (FileNotFoundException | NoSuchFileException e) { + // Ignore and return null } return null; } - private PGPPublicKey getPublicKey(KeyBlob blob, byte[] fingerprint) + private static PGPPublicKey getPublicKey(KeyBlob blob, byte[] fingerprint) throws IOException { return ((PublicKeyRingBlob) blob).getPGPPublicKeyRing() .getPublicKey(fingerprint); } - private PGPPublicKey getSigningPublicKey(KeyBlob blob) throws IOException { + private static PGPPublicKey getSigningPublicKey(KeyBlob blob) + throws IOException { PGPPublicKey masterKey = null; Iterator keys = ((PublicKeyRingBlob) blob) .getPGPPublicKeyRing().getPublicKeys(); @@ -618,7 +644,7 @@ private PGPPublicKey getSigningPublicKey(KeyBlob blob) throws IOException { return masterKey; } - private boolean isSigningKey(PGPPublicKey key) { + private static boolean isSigningKey(PGPPublicKey key) { Iterator signatures = key.getSignatures(); while (signatures.hasNext()) { PGPSignature sig = (PGPSignature) signatures.next(); @@ -630,7 +656,7 @@ private boolean isSigningKey(PGPPublicKey key) { return false; } - private KeyBox readKeyBoxFile(Path keyboxFile) throws IOException, + private static KeyBox readKeyBoxFile(Path keyboxFile) throws IOException, NoSuchAlgorithmException, NoSuchProviderException, NoOpenPgpKeyException { if (keyboxFile.toFile().length() == 0) { diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyPassphrasePrompt.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyPassphrasePrompt.java index e47f64f1a..614419598 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyPassphrasePrompt.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgKeyPassphrasePrompt.java @@ -17,8 +17,8 @@ import org.bouncycastle.util.encoders.Hex; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.transport.CredentialItem.CharArrayType; import org.eclipse.jgit.transport.CredentialItem.InformationalMessage; +import org.eclipse.jgit.transport.CredentialItem.Password; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.URIish; @@ -31,7 +31,7 @@ */ class BouncyCastleGpgKeyPassphrasePrompt implements AutoCloseable { - private CharArrayType passphrase; + private Password passphrase; private CredentialsProvider credentialsProvider; @@ -78,8 +78,7 @@ public char[] getPassphrase(byte[] keyFingerprint, Path keyLocation) throws PGPException, CanceledException, UnsupportedCredentialItem, URISyntaxException { if (passphrase == null) { - passphrase = new CharArrayType(BCText.get().credentialPassphrase, - true); + passphrase = new Password(BCText.get().credentialPassphrase); } if (credentialsProvider == null) { diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java new file mode 100644 index 000000000..7161895a6 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java @@ -0,0 +1,388 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.Security; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.Locale; + +import org.bouncycastle.bcpg.sig.IssuerFingerprint; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openpgp.PGPCompressedData; +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPSignature; +import org.bouncycastle.openpgp.PGPSignatureList; +import org.bouncycastle.openpgp.PGPSignatureSubpacketVector; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.jcajce.JcaPGPObjectFactory; +import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; +import org.bouncycastle.util.encoders.Hex; +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTag; +import org.eclipse.jgit.util.LRUMap; +import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.StringUtils; + +/** + * A {@link GpgSignatureVerifier} to verify GPG signatures using BouncyCastle. + */ +public class BouncyCastleGpgSignatureVerifier implements GpgSignatureVerifier { + + private static void registerBouncyCastleProviderIfNecessary() { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + /** + * Creates a new instance and registers the BouncyCastle security provider + * if needed. + */ + public BouncyCastleGpgSignatureVerifier() { + registerBouncyCastleProviderIfNecessary(); + } + + // To support more efficient signature verification of multiple objects we + // cache public keys once found in a LRU cache. + + private static final Object NO_KEY = new Object(); + + private LRUMap byFingerprint = new LRUMap<>(16, 200); + + private LRUMap bySigner = new LRUMap<>(16, 200); + + @Override + public String getName() { + return "bc"; //$NON-NLS-1$ + } + + @Override + @Nullable + public SignatureVerification verifySignature(@NonNull RevObject object, + @NonNull GpgConfig config) throws IOException { + if (object instanceof RevCommit) { + RevCommit commit = (RevCommit) object; + byte[] signatureData = commit.getRawGpgSignature(); + if (signatureData == null) { + return null; + } + byte[] raw = commit.getRawBuffer(); + // Now remove the GPG signature + byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; + int start = RawParseUtils.headerStart(header, raw, 0); + if (start < 0) { + return null; + } + int end = RawParseUtils.headerEnd(raw, start); + // start is at the beginning of the header's content + start -= header.length + 1; + // end is on the terminating LF; we need to skip that, too + if (end < raw.length) { + end++; + } + byte[] data = new byte[raw.length - (end - start)]; + System.arraycopy(raw, 0, data, 0, start); + System.arraycopy(raw, end, data, start, raw.length - end); + return verify(data, signatureData); + } else if (object instanceof RevTag) { + RevTag tag = (RevTag) object; + byte[] signatureData = tag.getRawGpgSignature(); + if (signatureData == null) { + return null; + } + byte[] raw = tag.getRawBuffer(); + // The signature is just tacked onto the end of the message, which + // is last in the buffer. + byte[] data = Arrays.copyOfRange(raw, 0, + raw.length - signatureData.length); + return verify(data, signatureData); + } + return null; + } + + static PGPSignature parseSignature(InputStream in) + throws IOException, PGPException { + try (InputStream sigIn = PGPUtil.getDecoderStream(in)) { + JcaPGPObjectFactory pgpFactory = new JcaPGPObjectFactory(sigIn); + Object obj = pgpFactory.nextObject(); + if (obj instanceof PGPCompressedData) { + obj = new JcaPGPObjectFactory( + ((PGPCompressedData) obj).getDataStream()).nextObject(); + } + if (obj instanceof PGPSignatureList) { + return ((PGPSignatureList) obj).get(0); + } + return null; + } + } + + @Override + public SignatureVerification verify(byte[] data, byte[] signatureData) + throws IOException { + PGPSignature signature = null; + String fingerprint = null; + String signer = null; + String keyId = null; + try (InputStream sigIn = new ByteArrayInputStream(signatureData)) { + signature = parseSignature(sigIn); + if (signature != null) { + // Try to figure out something to find the public key with. + if (signature.hasSubpackets()) { + PGPSignatureSubpacketVector packets = signature + .getHashedSubPackets(); + IssuerFingerprint fingerprintPacket = packets + .getIssuerFingerprint(); + if (fingerprintPacket != null) { + fingerprint = Hex + .toHexString(fingerprintPacket.getFingerprint()) + .toLowerCase(Locale.ROOT); + } + signer = packets.getSignerUserID(); + if (signer != null) { + signer = BouncyCastleGpgSigner.extractSignerId(signer); + } + } + keyId = Long.toUnsignedString(signature.getKeyID(), 16) + .toLowerCase(Locale.ROOT); + } else { + throw new JGitInternalException(BCText.get().nonSignatureError); + } + } catch (PGPException e) { + throw new JGitInternalException(BCText.get().signatureParseError, + e); + } + Date signatureCreatedAt = signature.getCreationTime(); + if (fingerprint == null && signer == null && keyId == null) { + return new VerificationResult(signatureCreatedAt, null, null, null, + false, false, TrustLevel.UNKNOWN, + BCText.get().signatureNoKeyInfo); + } + if (fingerprint != null && keyId != null + && !fingerprint.endsWith(keyId)) { + return new VerificationResult(signatureCreatedAt, signer, fingerprint, + null, false, false, TrustLevel.UNKNOWN, + MessageFormat.format(BCText.get().signatureInconsistent, + keyId, fingerprint)); + } + if (fingerprint == null && keyId != null) { + fingerprint = keyId; + } + // Try to find the public key + String keySpec = '<' + signer + '>'; + Object cached = null; + PGPPublicKey publicKey = null; + try { + cached = byFingerprint.get(fingerprint); + if (cached != null) { + if (cached instanceof PGPPublicKey) { + publicKey = (PGPPublicKey) cached; + } + } else if (!StringUtils.isEmptyOrNull(signer)) { + cached = bySigner.get(signer); + if (cached != null) { + if (cached instanceof PGPPublicKey) { + publicKey = (PGPPublicKey) cached; + } + } + } + if (cached == null) { + publicKey = BouncyCastleGpgKeyLocator.findPublicKey(fingerprint, + keySpec); + } + } catch (IOException | PGPException e) { + throw new JGitInternalException( + BCText.get().signatureKeyLookupError, e); + } + if (publicKey == null) { + if (cached == null) { + byFingerprint.put(fingerprint, NO_KEY); + byFingerprint.put(keyId, NO_KEY); + if (signer != null) { + bySigner.put(signer, NO_KEY); + } + } + return new VerificationResult(signatureCreatedAt, signer, + fingerprint, null, false, false, TrustLevel.UNKNOWN, + BCText.get().signatureNoPublicKey); + } + if (cached == null) { + byFingerprint.put(fingerprint, publicKey); + byFingerprint.put(keyId, publicKey); + if (signer != null) { + bySigner.put(signer, publicKey); + } + } + String user = null; + Iterator userIds = publicKey.getUserIDs(); + if (!StringUtils.isEmptyOrNull(signer)) { + while (userIds.hasNext()) { + String userId = userIds.next(); + if (BouncyCastleGpgKeyLocator.containsSigningKey(userId, + keySpec)) { + user = userId; + break; + } + } + } + if (user == null) { + userIds = publicKey.getUserIDs(); + if (userIds.hasNext()) { + user = userIds.next(); + } + } + boolean expired = false; + long validFor = publicKey.getValidSeconds(); + if (validFor > 0 && signatureCreatedAt != null) { + Instant expiredAt = publicKey.getCreationTime().toInstant() + .plusSeconds(validFor); + expired = expiredAt.isBefore(signatureCreatedAt.toInstant()); + } + // Trust data is not defined in OpenPGP; the format is implementation + // specific. We don't use the GPG trustdb but simply the trust packet + // on the public key, if present. Even if present, it may or may not + // be set. + byte[] trustData = publicKey.getTrustData(); + TrustLevel trust = parseGpgTrustPacket(trustData); + boolean verified = false; + try { + signature.init( + new JcaPGPContentVerifierBuilderProvider() + .setProvider(BouncyCastleProvider.PROVIDER_NAME), + publicKey); + signature.update(data); + verified = signature.verify(); + } catch (PGPException e) { + throw new JGitInternalException( + BCText.get().signatureVerificationError, e); + } + return new VerificationResult(signatureCreatedAt, signer, fingerprint, user, + verified, expired, trust, null); + } + + private TrustLevel parseGpgTrustPacket(byte[] packet) { + if (packet == null || packet.length < 6) { + // A GPG trust packet has at least 6 bytes. + return TrustLevel.UNKNOWN; + } + if (packet[2] != 'g' || packet[3] != 'p' || packet[4] != 'g') { + // Not a GPG trust packet + return TrustLevel.UNKNOWN; + } + int trust = packet[0] & 0x0F; + switch (trust) { + case 0: // No determined/set + case 1: // Trust expired; i.e., calculation outdated or key expired + case 2: // Undefined: not enough information to set + return TrustLevel.UNKNOWN; + case 3: + return TrustLevel.NEVER; + case 4: + return TrustLevel.MARGINAL; + case 5: + return TrustLevel.FULL; + case 6: + return TrustLevel.ULTIMATE; + default: + return TrustLevel.UNKNOWN; + } + } + + @Override + public void clear() { + byFingerprint.clear(); + bySigner.clear(); + } + + private static class VerificationResult implements SignatureVerification { + + private final Date creationDate; + + private final String signer; + + private final String keyUser; + + private final String fingerprint; + + private final boolean verified; + + private final boolean expired; + + private final @NonNull TrustLevel trustLevel; + + private final String message; + + public VerificationResult(Date creationDate, String signer, + String fingerprint, String user, boolean verified, + boolean expired, @NonNull TrustLevel trust, String message) { + this.creationDate = creationDate; + this.signer = signer; + this.fingerprint = fingerprint; + this.keyUser = user; + this.verified = verified; + this.expired = expired; + this.trustLevel = trust; + this.message = message; + } + + @Override + public Date getCreationDate() { + return creationDate; + } + + @Override + public String getSigner() { + return signer; + } + + @Override + public String getKeyUser() { + return keyUser; + } + + @Override + public String getKeyFingerprint() { + return fingerprint; + } + + @Override + public boolean isExpired() { + return expired; + } + + @Override + public TrustLevel getTrustLevel() { + return trustLevel; + } + + @Override + public String getMessage() { + return message; + } + + @Override + public boolean getVerified() { + return verified; + } + } +} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java new file mode 100644 index 000000000..ae82b758a --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal; + +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; + +/** + * A {@link GpgSignatureVerifierFactory} that creates + * {@link GpgSignatureVerifier} instances that verify GPG signatures using + * BouncyCastle and that do cache public keys. + */ +public final class BouncyCastleGpgSignatureVerifierFactory + extends GpgSignatureVerifierFactory { + + @Override + public GpgSignatureVerifier getVerifier() { + return new BouncyCastleGpgSignatureVerifier(); + } + +} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java index ea159c547..211bd7bd2 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, 2020, Salesforce and others + * Copyright (C) 2018, 2021, Salesforce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -34,18 +34,25 @@ import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.errors.UnsupportedCredentialItem; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgObjectSigner; import org.eclipse.jgit.lib.GpgSignature; import org.eclipse.jgit.lib.GpgSigner; +import org.eclipse.jgit.lib.ObjectBuilder; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.GpgConfig.GpgFormat; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.util.StringUtils; /** - * GPG Signer using BouncyCastle library + * GPG Signer using the BouncyCastle library. */ -public class BouncyCastleGpgSigner extends GpgSigner { +public class BouncyCastleGpgSigner extends GpgSigner + implements GpgObjectSigner { private static void registerBouncyCastleProviderIfNecessary() { if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { @@ -67,13 +74,32 @@ public BouncyCastleGpgSigner() { public boolean canLocateSigningKey(@Nullable String gpgSigningKey, PersonIdent committer, CredentialsProvider credentialsProvider) throws CanceledException { + try { + return canLocateSigningKey(gpgSigningKey, committer, + credentialsProvider, null); + } catch (UnsupportedSigningFormatException e) { + // Cannot occur with a null config + return false; + } + } + + @Override + public boolean canLocateSigningKey(@Nullable String gpgSigningKey, + PersonIdent committer, CredentialsProvider credentialsProvider, + GpgConfig config) + throws CanceledException, UnsupportedSigningFormatException { + if (config != null && config.getKeyFormat() != GpgFormat.OPENPGP) { + throw new UnsupportedSigningFormatException( + JGitText.get().onlyOpenPgpSupportedForSigning); + } try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( credentialsProvider)) { BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, committer, passphrasePrompt); return gpgKey != null; - } catch (PGPException | IOException | NoSuchAlgorithmException - | NoSuchProviderException | URISyntaxException e) { + } catch (CanceledException e) { + throw e; + } catch (Exception e) { return false; } } @@ -98,10 +124,28 @@ private BouncyCastleGpgKey locateSigningKey(@Nullable String gpgSigningKey, public void sign(@NonNull CommitBuilder commit, @Nullable String gpgSigningKey, @NonNull PersonIdent committer, CredentialsProvider credentialsProvider) throws CanceledException { + try { + signObject(commit, gpgSigningKey, committer, credentialsProvider, + null); + } catch (UnsupportedSigningFormatException e) { + // Cannot occur with a null config + } + } + + @Override + public void signObject(@NonNull ObjectBuilder object, + @Nullable String gpgSigningKey, @NonNull PersonIdent committer, + CredentialsProvider credentialsProvider, GpgConfig config) + throws CanceledException, UnsupportedSigningFormatException { + if (config != null && config.getKeyFormat() != GpgFormat.OPENPGP) { + throw new UnsupportedSigningFormatException( + JGitText.get().onlyOpenPgpSupportedForSigning); + } try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( credentialsProvider)) { BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, - committer, passphrasePrompt); + committer, + passphrasePrompt); PGPSecretKey secretKey = gpgKey.getSecretKey(); if (secretKey == null) { throw new JGitInternalException( @@ -158,17 +202,17 @@ public void sign(@NonNull CommitBuilder commit, ByteArrayOutputStream buffer = new ByteArrayOutputStream(); try (BCPGOutputStream out = new BCPGOutputStream( new ArmoredOutputStream(buffer))) { - signatureGenerator.update(commit.build()); + signatureGenerator.update(object.build()); signatureGenerator.generate().encode(out); } - commit.setGpgSignature(new GpgSignature(buffer.toByteArray())); + object.setGpgSignature(new GpgSignature(buffer.toByteArray())); } catch (PGPException | IOException | NoSuchAlgorithmException | NoSuchProviderException | URISyntaxException e) { throw new JGitInternalException(e.getMessage(), e); } } - private String extractSignerId(String pgpUserId) { + static String extractSignerId(String pgpUserId) { int from = pgpUserId.indexOf('<'); if (from >= 0) { int to = pgpUserId.indexOf('>', from + 1); diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/KeyGrip.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/KeyGrip.java new file mode 100644 index 000000000..b1d444601 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/KeyGrip.java @@ -0,0 +1,322 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Arrays; + +import org.bouncycastle.asn1.ASN1ObjectIdentifier; +import org.bouncycastle.asn1.cryptlib.CryptlibObjectIdentifiers; +import org.bouncycastle.asn1.x9.ECNamedCurveTable; +import org.bouncycastle.asn1.x9.X9ECParameters; +import org.bouncycastle.bcpg.DSAPublicBCPGKey; +import org.bouncycastle.bcpg.ECPublicBCPGKey; +import org.bouncycastle.bcpg.ElGamalPublicBCPGKey; +import org.bouncycastle.bcpg.PublicKeyAlgorithmTags; +import org.bouncycastle.bcpg.RSAPublicBCPGKey; +import org.bouncycastle.crypto.ec.CustomNamedCurves; +import org.bouncycastle.math.ec.ECAlgorithms; +import org.bouncycastle.math.field.FiniteField; +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.util.encoders.Hex; +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.gpg.bc.internal.BCText; +import org.eclipse.jgit.util.sha1.SHA1; + +/** + * Utilities to compute the keygrip of a key. A keygrip is a SHA1 hash + * over the public key parameters and is used internally by the gpg-agent to + * find the secret key belonging to a public key: the secret key is stored in a + * file under ~/.gnupg/private-keys-v1.d/ with a name "<keygrip>.key". While + * this storage organization is an implementation detail of GPG, the way + * keygrips are computed is not; they are computed by libgcrypt and their + * definition is stable. + */ +public final class KeyGrip { + + // Some OIDs apparently unknown to BouncyCastle. + + private static String OID_OPENPGP_ED25519 = "1.3.6.1.4.1.11591.15.1"; //$NON-NLS-1$ + + private static String OID_RFC8410_CURVE25519 = "1.3.101.110"; //$NON-NLS-1$ + + private static String OID_RFC8410_ED25519 = "1.3.101.112"; //$NON-NLS-1$ + + private KeyGrip() { + // No instantiation + } + + /** + * Computes the keygrip for a {@link PGPPublicKey}. + * + * @param publicKey + * to get the keygrip of + * @return the keygrip + * @throws PGPException + * if an unknown key type is encountered. + */ + @NonNull + public static byte[] getKeyGrip(PGPPublicKey publicKey) + throws PGPException { + SHA1 grip = SHA1.newInstance(); + grip.setDetectCollision(false); + + switch (publicKey.getAlgorithm()) { + case PublicKeyAlgorithmTags.RSA_GENERAL: + case PublicKeyAlgorithmTags.RSA_ENCRYPT: + case PublicKeyAlgorithmTags.RSA_SIGN: + BigInteger modulus = ((RSAPublicBCPGKey) publicKey + .getPublicKeyPacket().getKey()).getModulus(); + hash(grip, modulus.toByteArray()); + break; + case PublicKeyAlgorithmTags.DSA: + DSAPublicBCPGKey dsa = (DSAPublicBCPGKey) publicKey + .getPublicKeyPacket().getKey(); + hash(grip, dsa.getP().toByteArray(), 'p', true); + hash(grip, dsa.getQ().toByteArray(), 'q', true); + hash(grip, dsa.getG().toByteArray(), 'g', true); + hash(grip, dsa.getY().toByteArray(), 'y', true); + break; + case PublicKeyAlgorithmTags.ELGAMAL_GENERAL: + case PublicKeyAlgorithmTags.ELGAMAL_ENCRYPT: + ElGamalPublicBCPGKey eg = (ElGamalPublicBCPGKey) publicKey + .getPublicKeyPacket().getKey(); + hash(grip, eg.getP().toByteArray(), 'p', true); + hash(grip, eg.getG().toByteArray(), 'g', true); + hash(grip, eg.getY().toByteArray(), 'y', true); + break; + case PublicKeyAlgorithmTags.ECDH: + case PublicKeyAlgorithmTags.ECDSA: + case PublicKeyAlgorithmTags.EDDSA: + ECPublicBCPGKey ec = (ECPublicBCPGKey) publicKey + .getPublicKeyPacket().getKey(); + ASN1ObjectIdentifier curveOID = ec.getCurveOID(); + // BC doesn't know these OIDs. + if (OID_OPENPGP_ED25519.equals(curveOID.getId()) + || OID_RFC8410_ED25519.equals(curveOID.getId())) { + return hashEd25519(grip, ec.getEncodedPoint()); + } else if (CryptlibObjectIdentifiers.curvey25519.equals(curveOID) + || OID_RFC8410_CURVE25519.equals(curveOID.getId())) { + // curvey25519 actually is the OpenPGP OID for Curve25519 and is + // known to BC, but the parameters are for the short Weierstrass + // form. See https://github.com/bcgit/bc-java/issues/399 . + // libgcrypt uses Montgomery form. + return hashCurve25519(grip, ec.getEncodedPoint()); + } + X9ECParameters params = getX9Parameters(curveOID); + if (params == null) { + throw new PGPException(MessageFormat + .format(BCText.get().unknownCurve, curveOID.getId())); + } + // Need to write p, a, b, g, n, q + BigInteger q = ec.getEncodedPoint(); + byte[] g = params.getG().getEncoded(false); + BigInteger a = params.getCurve().getA().toBigInteger(); + BigInteger b = params.getCurve().getB().toBigInteger(); + BigInteger n = params.getN(); + BigInteger p = null; + FiniteField field = params.getCurve().getField(); + if (ECAlgorithms.isFpField(field)) { + p = field.getCharacteristic(); + } + if (p == null) { + // Don't know... + throw new PGPException(MessageFormat.format( + BCText.get().unknownCurveParameters, curveOID.getId())); + } + hash(grip, p.toByteArray(), 'p', false); + hash(grip, a.toByteArray(), 'a', false); + hash(grip, b.toByteArray(), 'b', false); + hash(grip, g, 'g', false); + hash(grip, n.toByteArray(), 'n', false); + if (publicKey.getAlgorithm() == PublicKeyAlgorithmTags.EDDSA) { + hashQ25519(grip, q); + } else { + hash(grip, q.toByteArray(), 'q', false); + } + break; + default: + throw new PGPException( + MessageFormat.format(BCText.get().unknownKeyType, + Integer.toString(publicKey.getAlgorithm()))); + } + return grip.digest(); + } + + private static void hash(SHA1 grip, byte[] data) { + // Need to skip leading zero bytes + int i = 0; + while (i < data.length && data[i] == 0) { + i++; + } + int length = data.length - i; + if (i < data.length) { + if ((data[i] & 0x80) != 0) { + grip.update((byte) 0); + } + grip.update(data, i, length); + } + } + + private static void hash(SHA1 grip, byte[] data, char id, boolean zeroPad) { + // Need to skip leading zero bytes + int i = 0; + while (i < data.length && data[i] == 0) { + i++; + } + int length = data.length - i; + boolean addZero = false; + if (i < data.length && zeroPad && (data[i] & 0x80) != 0) { + addZero = true; + } + // libgcrypt includes an SExp in the hash + String prefix = "(1:" + id + (addZero ? length + 1 : length) + ':'; //$NON-NLS-1$ + grip.update(prefix.getBytes(StandardCharsets.US_ASCII)); + // For some items, gcrypt prepends a zero byte if the high bit is set + if (addZero) { + grip.update((byte) 0); + } + if (i < data.length) { + grip.update(data, i, length); + } + grip.update((byte) ')'); + } + + private static void hashQ25519(SHA1 grip, BigInteger q) + throws PGPException { + byte[] data = q.toByteArray(); + switch (data[0]) { + case 0x04: + if (data.length != 65) { + throw new PGPException(MessageFormat.format( + BCText.get().corrupt25519Key, Hex.toHexString(data))); + } + // Uncompressed: should not occur with ed25519 or curve25519 + throw new PGPException(MessageFormat.format( + BCText.get().uncompressed25519Key, Hex.toHexString(data))); + case 0x40: + if (data.length != 33) { + throw new PGPException(MessageFormat.format( + BCText.get().corrupt25519Key, Hex.toHexString(data))); + } + // Compressed; normal case. Skip prefix. + hash(grip, Arrays.copyOfRange(data, 1, data.length), 'q', false); + break; + default: + if (data.length != 32) { + throw new PGPException(MessageFormat.format( + BCText.get().corrupt25519Key, Hex.toHexString(data))); + } + // Compressed format without prefix. Should not occur? + hash(grip, data, 'q', false); + break; + } + } + + /** + * Computes the keygrip for an ed25519 public key. + *

    + * Package-visible for tests only. + *

    + * + * @param grip + * initialized {@link SHA1} + * @param q + * the public key's EC point + * @return the keygrip + * @throws PGPException + * if q indicates uncompressed format + */ + @SuppressWarnings("nls") + static byte[] hashEd25519(SHA1 grip, BigInteger q) throws PGPException { + // For the values, see RFC 7748: https://tools.ietf.org/html/rfc7748 + // p = 2^255 - 19 + hash(grip, Hex.decodeStrict( + "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFED"), + 'p', false); + // Field: a = 1 + hash(grip, new byte[] { 0x01 }, 'a', false); + // Field: b = 121665/121666 (mod p) + // See Berstein et.al., "Twisted Edwards Curves", + // https://doi.org/10.1007/978-3-540-68164-9_26 + hash(grip, Hex.decodeStrict( + "2DFC9311D490018C7338BF8688861767FF8FF5B2BEBE27548A14B235ECA6874A"), + 'b', false); + // Generator point with affine X,Y + // @formatter:off + // X(P) = 15112221349535400772501151409588531511454012693041857206046113283949847762202 + // Y(P) = 46316835694926478169428394003475163141307993866256225615783033603165251855960 + // the "04" signifies uncompressed format. + // @formatter:on + hash(grip, Hex.decodeStrict("04" + + "216936D3CD6E53FEC0A4E231FDD6DC5C692CC7609525A7B2C9562D608F25D51A" + + "6666666666666666666666666666666666666666666666666666666666666658"), + 'g', false); + // order = 2^252 + 0x14def9dea2f79cd65812631a5cf5d3ed + hash(grip, Hex.decodeStrict( + "1000000000000000000000000000000014DEF9DEA2F79CD65812631A5CF5D3ED"), + 'n', false); + hashQ25519(grip, q); + return grip.digest(); + } + + /** + * Computes the keygrip for a curve25519 public key. + *

    + * Package-visible for tests only. + *

    + * + * @param grip + * initialized {@link SHA1} + * @param q + * the public key's EC point + * @return the keygrip + * @throws PGPException + * if q indicates uncompressed format + */ + @SuppressWarnings("nls") + static byte[] hashCurve25519(SHA1 grip, BigInteger q) throws PGPException { + hash(grip, Hex.decodeStrict( + "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFED"), + 'p', false); + // Unclear: RFC 7748 says A = 486662. This value here is (A-2)/4 = + // 121665. Compare ecc-curves.c in libgcrypt: + // https://github.com/gpg/libgcrypt/blob/361a058/cipher/ecc-curves.c#L146 + hash(grip, new byte[] { 0x01, (byte) 0xDB, 0x41 }, 'a', false); + hash(grip, new byte[] { 0x01 }, 'b', false); + // libgcrypt uses the old g.y value before the erratum to RFC 7748 for + // the keygrip. The new value would be + // 5F51E65E475F794B1FE122D388B72EB36DC2B28192839E4DD6163A5D81312C14. See + // https://www.rfc-editor.org/errata/eid4730 and + // https://github.com/gpg/libgcrypt/commit/f67b6492e0b0 + hash(grip, Hex.decodeStrict("04" + + "0000000000000000000000000000000000000000000000000000000000000009" + + "20AE19A1B8A086B4E01EDD2C7748D14C923D4D7E6D7C61B229E9C5A27ECED3D9"), + 'g', false); + hash(grip, Hex.decodeStrict( + "1000000000000000000000000000000014DEF9DEA2F79CD65812631A5CF5D3ED"), + 'n', false); + hashQ25519(grip, q); + return grip.digest(); + } + + private static X9ECParameters getX9Parameters( + ASN1ObjectIdentifier curveOID) { + X9ECParameters params = CustomNamedCurves.getByOID(curveOID); + if (params == null) { + params = ECNamedCurveTable.getByOID(curveOID); + } + return params; + } + +} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/OCBPBEProtectionRemoverFactory.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/OCBPBEProtectionRemoverFactory.java new file mode 100644 index 000000000..68f8a4555 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/OCBPBEProtectionRemoverFactory.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import java.security.NoSuchAlgorithmException; +import java.text.MessageFormat; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPUtil; +import org.bouncycastle.openpgp.operator.PBEProtectionRemoverFactory; +import org.bouncycastle.openpgp.operator.PBESecretKeyDecryptor; +import org.bouncycastle.openpgp.operator.PGPDigestCalculatorProvider; +import org.bouncycastle.util.Arrays; +import org.eclipse.jgit.gpg.bc.internal.BCText; + +/** + * A {@link PBEProtectionRemoverFactory} using AES/OCB/NoPadding for decryption. + * It accepts an AAD in the factory's constructor, so the factory can be used to + * create a {@link PBESecretKeyDecryptor} only for a particular input. + *

    + * For JGit's needs, this is sufficient, but for a general upstream + * implementation that limitation might not be acceptable. + *

    + */ +class OCBPBEProtectionRemoverFactory + implements PBEProtectionRemoverFactory { + + private final PGPDigestCalculatorProvider calculatorProvider; + + private final char[] passphrase; + + private final byte[] aad; + + /** + * Creates a new factory instance with the given parameters. + *

    + * Because the AAD is given at factory level, the {@link PBESecretKeyDecryptor}s + * created by the factory can be used to decrypt only a particular input + * matching this AAD. + *

    + * + * @param passphrase to use for secret key derivation + * @param calculatorProvider for computing digests + * @param aad for the OCB decryption + */ + OCBPBEProtectionRemoverFactory(char[] passphrase, + PGPDigestCalculatorProvider calculatorProvider, byte[] aad) { + this.calculatorProvider = calculatorProvider; + this.passphrase = passphrase; + this.aad = aad; + } + + @Override + public PBESecretKeyDecryptor createDecryptor(String protection) + throws PGPException { + return new PBESecretKeyDecryptor(passphrase, calculatorProvider) { + + @Override + public byte[] recoverKeyData(int encAlgorithm, byte[] key, + byte[] iv, byte[] encrypted, int encryptedOffset, + int encryptedLength) throws PGPException { + String algorithmName = PGPUtil + .getSymmetricCipherName(encAlgorithm); + byte[] decrypted = null; + try { + Cipher c = Cipher + .getInstance(algorithmName + "/OCB/NoPadding"); //$NON-NLS-1$ + SecretKey secretKey = new SecretKeySpec(key, algorithmName); + c.init(Cipher.DECRYPT_MODE, secretKey, + new IvParameterSpec(iv)); + c.updateAAD(aad); + decrypted = new byte[c.getOutputSize(encryptedLength)]; + int decryptedLength = c.update(encrypted, encryptedOffset, + encryptedLength, decrypted); + // doFinal() for OCB will check the MAC and throw an + // exception if it doesn't match + decryptedLength += c.doFinal(decrypted, decryptedLength); + if (decryptedLength != decrypted.length) { + throw new PGPException(MessageFormat.format( + BCText.get().cryptWrongDecryptedLength, + Integer.valueOf(decryptedLength), + Integer.valueOf(decrypted.length))); + } + byte[] result = decrypted; + decrypted = null; // Don't clear in finally + return result; + } catch (NoClassDefFoundError e) { + String msg = MessageFormat.format( + BCText.get().gpgNoSuchAlgorithm, + algorithmName + "/OCB"); //$NON-NLS-1$ + throw new PGPException(msg, + new NoSuchAlgorithmException(msg, e)); + } catch (PGPException e) { + throw e; + } catch (Exception e) { + throw new PGPException( + MessageFormat.format(BCText.get().cryptCipherError, + e.getLocalizedMessage()), + e); + } finally { + if (decrypted != null) { + // Prevent halfway decrypted data leaking. + Arrays.fill(decrypted, (byte) 0); + } + } + } + }; + } +} \ No newline at end of file diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SExprParser.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SExprParser.java new file mode 100644 index 000000000..a9bb22c78 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SExprParser.java @@ -0,0 +1,826 @@ +/* + * Copyright (c) 2000-2021 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + *

    + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without restriction, + *including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + *

    + *

    + * The above copyright notice and this permission notice shall be included in all copies or substantial + * portions of the Software. + *

    + *

    + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *

    + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.util.Date; + +import org.bouncycastle.asn1.x9.ECNamedCurveTable; +import org.bouncycastle.bcpg.DSAPublicBCPGKey; +import org.bouncycastle.bcpg.DSASecretBCPGKey; +import org.bouncycastle.bcpg.ECDSAPublicBCPGKey; +import org.bouncycastle.bcpg.ECPublicBCPGKey; +import org.bouncycastle.bcpg.ECSecretBCPGKey; +import org.bouncycastle.bcpg.ElGamalPublicBCPGKey; +import org.bouncycastle.bcpg.ElGamalSecretBCPGKey; +import org.bouncycastle.bcpg.HashAlgorithmTags; +import org.bouncycastle.bcpg.PublicKeyAlgorithmTags; +import org.bouncycastle.bcpg.PublicKeyPacket; +import org.bouncycastle.bcpg.RSAPublicBCPGKey; +import org.bouncycastle.bcpg.RSASecretBCPGKey; +import org.bouncycastle.bcpg.S2K; +import org.bouncycastle.bcpg.SecretKeyPacket; +import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags; +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPSecretKey; +import org.bouncycastle.openpgp.operator.KeyFingerPrintCalculator; +import org.bouncycastle.openpgp.operator.PBEProtectionRemoverFactory; +import org.bouncycastle.openpgp.operator.PBESecretKeyDecryptor; +import org.bouncycastle.openpgp.operator.PGPDigestCalculator; +import org.bouncycastle.openpgp.operator.PGPDigestCalculatorProvider; +import org.bouncycastle.util.Arrays; +import org.bouncycastle.util.Strings; + +/** + * A parser for secret keys stored in s-expressions. Original BouncyCastle code + * modified by the JGit team to: + *
      + *
    • handle unencrypted DSA, EC, and ElGamal keys (upstream only handles + * unencrypted RSA), and
      • + *
    • handle secret keys using AES/OCB as encryption (those don't have a + * hash).
      • + *
    + */ +@SuppressWarnings("nls") +public class SExprParser { + private final PGPDigestCalculatorProvider digestProvider; + + /** + * Base constructor. + * + * @param digestProvider + * a provider for digest calculations. Used to confirm key + * protection hashes. + */ + public SExprParser(PGPDigestCalculatorProvider digestProvider) { + this.digestProvider = digestProvider; + } + + /** + * Parse a secret key from one of the GPG S expression keys associating it + * with the passed in public key. + * + * @param inputStream + * to read from + * @param keyProtectionRemoverFactory + * for decrypting encrypted keys + * @param pubKey + * the private key should belong to + * + * @return a secret key object. + * @throws IOException + * @throws PGPException + */ + public PGPSecretKey parseSecretKey(InputStream inputStream, + PBEProtectionRemoverFactory keyProtectionRemoverFactory, + PGPPublicKey pubKey) throws IOException, PGPException { + SXprUtils.skipOpenParenthesis(inputStream); + + String type; + + type = SXprUtils.readString(inputStream, inputStream.read()); + if (type.equals("protected-private-key") + || type.equals("private-key")) { + SXprUtils.skipOpenParenthesis(inputStream); + + String keyType = SXprUtils.readString(inputStream, + inputStream.read()); + if (keyType.equals("ecc")) { + SXprUtils.skipOpenParenthesis(inputStream); + + String curveID = SXprUtils.readString(inputStream, + inputStream.read()); + String curveName = SXprUtils.readString(inputStream, + inputStream.read()); + + SXprUtils.skipCloseParenthesis(inputStream); + + byte[] qVal; + + SXprUtils.skipOpenParenthesis(inputStream); + + type = SXprUtils.readString(inputStream, inputStream.read()); + if (type.equals("q")) { + qVal = SXprUtils.readBytes(inputStream, inputStream.read()); + } else { + throw new PGPException("no q value found"); + } + + SXprUtils.skipCloseParenthesis(inputStream); + + BigInteger d = processECSecretKey(inputStream, curveID, + curveName, qVal, keyProtectionRemoverFactory); + + if (curveName.startsWith("NIST ")) { + curveName = curveName.substring("NIST ".length()); + } + + ECPublicBCPGKey basePubKey = new ECDSAPublicBCPGKey( + ECNamedCurveTable.getOID(curveName), + new BigInteger(1, qVal)); + ECPublicBCPGKey assocPubKey = (ECPublicBCPGKey) pubKey + .getPublicKeyPacket().getKey(); + if (!basePubKey.getCurveOID().equals(assocPubKey.getCurveOID()) + || !basePubKey.getEncodedPoint() + .equals(assocPubKey.getEncodedPoint())) { + throw new PGPException( + "passed in public key does not match secret key"); + } + + return new PGPSecretKey( + new SecretKeyPacket(pubKey.getPublicKeyPacket(), + SymmetricKeyAlgorithmTags.NULL, null, null, + new ECSecretBCPGKey(d).getEncoded()), + pubKey); + } else if (keyType.equals("dsa")) { + BigInteger p = readBigInteger("p", inputStream); + BigInteger q = readBigInteger("q", inputStream); + BigInteger g = readBigInteger("g", inputStream); + + BigInteger y = readBigInteger("y", inputStream); + + BigInteger x = processDSASecretKey(inputStream, p, q, g, y, + keyProtectionRemoverFactory); + + DSAPublicBCPGKey basePubKey = new DSAPublicBCPGKey(p, q, g, y); + DSAPublicBCPGKey assocPubKey = (DSAPublicBCPGKey) pubKey + .getPublicKeyPacket().getKey(); + if (!basePubKey.getP().equals(assocPubKey.getP()) + || !basePubKey.getQ().equals(assocPubKey.getQ()) + || !basePubKey.getG().equals(assocPubKey.getG()) + || !basePubKey.getY().equals(assocPubKey.getY())) { + throw new PGPException( + "passed in public key does not match secret key"); + } + return new PGPSecretKey( + new SecretKeyPacket(pubKey.getPublicKeyPacket(), + SymmetricKeyAlgorithmTags.NULL, null, null, + new DSASecretBCPGKey(x).getEncoded()), + pubKey); + } else if (keyType.equals("elg")) { + BigInteger p = readBigInteger("p", inputStream); + BigInteger g = readBigInteger("g", inputStream); + + BigInteger y = readBigInteger("y", inputStream); + + BigInteger x = processElGamalSecretKey(inputStream, p, g, y, + keyProtectionRemoverFactory); + + ElGamalPublicBCPGKey basePubKey = new ElGamalPublicBCPGKey(p, g, + y); + ElGamalPublicBCPGKey assocPubKey = (ElGamalPublicBCPGKey) pubKey + .getPublicKeyPacket().getKey(); + if (!basePubKey.getP().equals(assocPubKey.getP()) + || !basePubKey.getG().equals(assocPubKey.getG()) + || !basePubKey.getY().equals(assocPubKey.getY())) { + throw new PGPException( + "passed in public key does not match secret key"); + } + + return new PGPSecretKey( + new SecretKeyPacket(pubKey.getPublicKeyPacket(), + SymmetricKeyAlgorithmTags.NULL, null, null, + new ElGamalSecretBCPGKey(x).getEncoded()), + pubKey); + } else if (keyType.equals("rsa")) { + BigInteger n = readBigInteger("n", inputStream); + BigInteger e = readBigInteger("e", inputStream); + + BigInteger[] values = processRSASecretKey(inputStream, n, e, + keyProtectionRemoverFactory); + + // TODO: type of RSA key? + RSAPublicBCPGKey basePubKey = new RSAPublicBCPGKey(n, e); + RSAPublicBCPGKey assocPubKey = (RSAPublicBCPGKey) pubKey + .getPublicKeyPacket().getKey(); + if (!basePubKey.getModulus().equals(assocPubKey.getModulus()) + || !basePubKey.getPublicExponent() + .equals(assocPubKey.getPublicExponent())) { + throw new PGPException( + "passed in public key does not match secret key"); + } + + return new PGPSecretKey(new SecretKeyPacket( + pubKey.getPublicKeyPacket(), + SymmetricKeyAlgorithmTags.NULL, null, null, + new RSASecretBCPGKey(values[0], values[1], values[2]) + .getEncoded()), + pubKey); + } else { + throw new PGPException("unknown key type: " + keyType); + } + } + + throw new PGPException("unknown key type found"); + } + + /** + * Parse a secret key from one of the GPG S expression keys. + * + * @param inputStream + * to read from + * @param keyProtectionRemoverFactory + * for decrypting encrypted keys + * @param fingerPrintCalculator + * for calculating key fingerprints + * + * @return a secret key object. + * @throws IOException + * @throws PGPException + */ + public PGPSecretKey parseSecretKey(InputStream inputStream, + PBEProtectionRemoverFactory keyProtectionRemoverFactory, + KeyFingerPrintCalculator fingerPrintCalculator) + throws IOException, PGPException { + SXprUtils.skipOpenParenthesis(inputStream); + + String type; + + type = SXprUtils.readString(inputStream, inputStream.read()); + if (type.equals("protected-private-key") + || type.equals("private-key")) { + SXprUtils.skipOpenParenthesis(inputStream); + + String keyType = SXprUtils.readString(inputStream, + inputStream.read()); + if (keyType.equals("ecc")) { + SXprUtils.skipOpenParenthesis(inputStream); + + String curveID = SXprUtils.readString(inputStream, + inputStream.read()); + String curveName = SXprUtils.readString(inputStream, + inputStream.read()); + + if (curveName.startsWith("NIST ")) { + curveName = curveName.substring("NIST ".length()); + } + + SXprUtils.skipCloseParenthesis(inputStream); + + byte[] qVal; + + SXprUtils.skipOpenParenthesis(inputStream); + + type = SXprUtils.readString(inputStream, inputStream.read()); + if (type.equals("q")) { + qVal = SXprUtils.readBytes(inputStream, inputStream.read()); + } else { + throw new PGPException("no q value found"); + } + + PublicKeyPacket pubPacket = new PublicKeyPacket( + PublicKeyAlgorithmTags.ECDSA, new Date(), + new ECDSAPublicBCPGKey( + ECNamedCurveTable.getOID(curveName), + new BigInteger(1, qVal))); + + SXprUtils.skipCloseParenthesis(inputStream); + + BigInteger d = processECSecretKey(inputStream, curveID, + curveName, qVal, keyProtectionRemoverFactory); + + return new PGPSecretKey( + new SecretKeyPacket(pubPacket, + SymmetricKeyAlgorithmTags.NULL, null, null, + new ECSecretBCPGKey(d).getEncoded()), + new PGPPublicKey(pubPacket, fingerPrintCalculator)); + } else if (keyType.equals("dsa")) { + BigInteger p = readBigInteger("p", inputStream); + BigInteger q = readBigInteger("q", inputStream); + BigInteger g = readBigInteger("g", inputStream); + + BigInteger y = readBigInteger("y", inputStream); + + BigInteger x = processDSASecretKey(inputStream, p, q, g, y, + keyProtectionRemoverFactory); + + PublicKeyPacket pubPacket = new PublicKeyPacket( + PublicKeyAlgorithmTags.DSA, new Date(), + new DSAPublicBCPGKey(p, q, g, y)); + + return new PGPSecretKey( + new SecretKeyPacket(pubPacket, + SymmetricKeyAlgorithmTags.NULL, null, null, + new DSASecretBCPGKey(x).getEncoded()), + new PGPPublicKey(pubPacket, fingerPrintCalculator)); + } else if (keyType.equals("elg")) { + BigInteger p = readBigInteger("p", inputStream); + BigInteger g = readBigInteger("g", inputStream); + + BigInteger y = readBigInteger("y", inputStream); + + BigInteger x = processElGamalSecretKey(inputStream, p, g, y, + keyProtectionRemoverFactory); + + PublicKeyPacket pubPacket = new PublicKeyPacket( + PublicKeyAlgorithmTags.ELGAMAL_ENCRYPT, new Date(), + new ElGamalPublicBCPGKey(p, g, y)); + + return new PGPSecretKey( + new SecretKeyPacket(pubPacket, + SymmetricKeyAlgorithmTags.NULL, null, null, + new ElGamalSecretBCPGKey(x).getEncoded()), + new PGPPublicKey(pubPacket, fingerPrintCalculator)); + } else if (keyType.equals("rsa")) { + BigInteger n = readBigInteger("n", inputStream); + BigInteger e = readBigInteger("e", inputStream); + + BigInteger[] values = processRSASecretKey(inputStream, n, e, + keyProtectionRemoverFactory); + + // TODO: type of RSA key? + PublicKeyPacket pubPacket = new PublicKeyPacket( + PublicKeyAlgorithmTags.RSA_GENERAL, new Date(), + new RSAPublicBCPGKey(n, e)); + + return new PGPSecretKey( + new SecretKeyPacket(pubPacket, + SymmetricKeyAlgorithmTags.NULL, null, null, + new RSASecretBCPGKey(values[0], values[1], + values[2]).getEncoded()), + new PGPPublicKey(pubPacket, fingerPrintCalculator)); + } else { + throw new PGPException("unknown key type: " + keyType); + } + } + + throw new PGPException("unknown key type found"); + } + + private BigInteger readBigInteger(String expectedType, + InputStream inputStream) throws IOException, PGPException { + SXprUtils.skipOpenParenthesis(inputStream); + + String type = SXprUtils.readString(inputStream, inputStream.read()); + if (!type.equals(expectedType)) { + throw new PGPException(expectedType + " value expected"); + } + + byte[] nBytes = SXprUtils.readBytes(inputStream, inputStream.read()); + BigInteger v = new BigInteger(1, nBytes); + + SXprUtils.skipCloseParenthesis(inputStream); + + return v; + } + + private static byte[][] extractData(InputStream inputStream, + PBEProtectionRemoverFactory keyProtectionRemoverFactory) + throws PGPException, IOException { + byte[] data; + byte[] protectedAt = null; + + SXprUtils.skipOpenParenthesis(inputStream); + + String type = SXprUtils.readString(inputStream, inputStream.read()); + if (type.equals("protected")) { + String protection = SXprUtils.readString(inputStream, + inputStream.read()); + + SXprUtils.skipOpenParenthesis(inputStream); + + S2K s2k = SXprUtils.parseS2K(inputStream); + + byte[] iv = SXprUtils.readBytes(inputStream, inputStream.read()); + + SXprUtils.skipCloseParenthesis(inputStream); + + byte[] secKeyData = SXprUtils.readBytes(inputStream, + inputStream.read()); + + SXprUtils.skipCloseParenthesis(inputStream); + + PBESecretKeyDecryptor keyDecryptor = keyProtectionRemoverFactory + .createDecryptor(protection); + + // TODO: recognise other algorithms + byte[] key = keyDecryptor.makeKeyFromPassPhrase( + SymmetricKeyAlgorithmTags.AES_128, s2k); + + data = keyDecryptor.recoverKeyData( + SymmetricKeyAlgorithmTags.AES_128, key, iv, secKeyData, 0, + secKeyData.length); + + // check if protected at is present + if (inputStream.read() == '(') { + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + + bOut.write('('); + int ch; + while ((ch = inputStream.read()) >= 0 && ch != ')') { + bOut.write(ch); + } + + if (ch != ')') { + throw new IOException("unexpected end to SExpr"); + } + + bOut.write(')'); + + protectedAt = bOut.toByteArray(); + } + + SXprUtils.skipCloseParenthesis(inputStream); + SXprUtils.skipCloseParenthesis(inputStream); + } else if (type.equals("d") || type.equals("x")) { + // JGit modification: unencrypted DSA or ECC keys can have an "x" + // here + return null; + } else { + throw new PGPException("protected block not found"); + } + + return new byte[][] { data, protectedAt }; + } + + private BigInteger processDSASecretKey(InputStream inputStream, + BigInteger p, BigInteger q, BigInteger g, BigInteger y, + PBEProtectionRemoverFactory keyProtectionRemoverFactory) + throws IOException, PGPException { + String type; + byte[][] basicData = extractData(inputStream, + keyProtectionRemoverFactory); + + // JGit modification: handle unencrypted DSA keys + if (basicData == null) { + byte[] nBytes = SXprUtils.readBytes(inputStream, + inputStream.read()); + BigInteger x = new BigInteger(1, nBytes); + SXprUtils.skipCloseParenthesis(inputStream); + return x; + } + + byte[] keyData = basicData[0]; + byte[] protectedAt = basicData[1]; + + // + // parse the secret key S-expr + // + InputStream keyIn = new ByteArrayInputStream(keyData); + + SXprUtils.skipOpenParenthesis(keyIn); + SXprUtils.skipOpenParenthesis(keyIn); + + BigInteger x = readBigInteger("x", keyIn); + + SXprUtils.skipCloseParenthesis(keyIn); + + // JGit modification: OCB-encrypted keys don't have and don't need a + // hash + if (keyProtectionRemoverFactory instanceof OCBPBEProtectionRemoverFactory) { + return x; + } + + SXprUtils.skipOpenParenthesis(keyIn); + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("hash")) { + throw new PGPException("hash keyword expected"); + } + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("sha1")) { + throw new PGPException("hash keyword expected"); + } + + byte[] hashBytes = SXprUtils.readBytes(keyIn, keyIn.read()); + + SXprUtils.skipCloseParenthesis(keyIn); + + if (digestProvider != null) { + PGPDigestCalculator digestCalculator = digestProvider + .get(HashAlgorithmTags.SHA1); + + OutputStream dOut = digestCalculator.getOutputStream(); + + dOut.write(Strings.toByteArray("(3:dsa")); + writeCanonical(dOut, "p", p); + writeCanonical(dOut, "q", q); + writeCanonical(dOut, "g", g); + writeCanonical(dOut, "y", y); + writeCanonical(dOut, "x", x); + + // check protected-at + if (protectedAt != null) { + dOut.write(protectedAt); + } + + dOut.write(Strings.toByteArray(")")); + + byte[] check = digestCalculator.getDigest(); + if (!Arrays.constantTimeAreEqual(check, hashBytes)) { + throw new PGPException( + "checksum on protected data failed in SExpr"); + } + } + + return x; + } + + private BigInteger processElGamalSecretKey(InputStream inputStream, + BigInteger p, BigInteger g, BigInteger y, + PBEProtectionRemoverFactory keyProtectionRemoverFactory) + throws IOException, PGPException { + String type; + byte[][] basicData = extractData(inputStream, + keyProtectionRemoverFactory); + + // JGit modification: handle unencrypted EC keys + if (basicData == null) { + byte[] nBytes = SXprUtils.readBytes(inputStream, + inputStream.read()); + BigInteger x = new BigInteger(1, nBytes); + SXprUtils.skipCloseParenthesis(inputStream); + return x; + } + + byte[] keyData = basicData[0]; + byte[] protectedAt = basicData[1]; + + // + // parse the secret key S-expr + // + InputStream keyIn = new ByteArrayInputStream(keyData); + + SXprUtils.skipOpenParenthesis(keyIn); + SXprUtils.skipOpenParenthesis(keyIn); + + BigInteger x = readBigInteger("x", keyIn); + + SXprUtils.skipCloseParenthesis(keyIn); + + // JGit modification: OCB-encrypted keys don't have and don't need a + // hash + if (keyProtectionRemoverFactory instanceof OCBPBEProtectionRemoverFactory) { + return x; + } + + SXprUtils.skipOpenParenthesis(keyIn); + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("hash")) { + throw new PGPException("hash keyword expected"); + } + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("sha1")) { + throw new PGPException("hash keyword expected"); + } + + byte[] hashBytes = SXprUtils.readBytes(keyIn, keyIn.read()); + + SXprUtils.skipCloseParenthesis(keyIn); + + if (digestProvider != null) { + PGPDigestCalculator digestCalculator = digestProvider + .get(HashAlgorithmTags.SHA1); + + OutputStream dOut = digestCalculator.getOutputStream(); + + dOut.write(Strings.toByteArray("(3:elg")); + writeCanonical(dOut, "p", p); + writeCanonical(dOut, "g", g); + writeCanonical(dOut, "y", y); + writeCanonical(dOut, "x", x); + + // check protected-at + if (protectedAt != null) { + dOut.write(protectedAt); + } + + dOut.write(Strings.toByteArray(")")); + + byte[] check = digestCalculator.getDigest(); + if (!Arrays.constantTimeAreEqual(check, hashBytes)) { + throw new PGPException( + "checksum on protected data failed in SExpr"); + } + } + + return x; + } + + private BigInteger processECSecretKey(InputStream inputStream, + String curveID, String curveName, byte[] qVal, + PBEProtectionRemoverFactory keyProtectionRemoverFactory) + throws IOException, PGPException { + String type; + + byte[][] basicData = extractData(inputStream, + keyProtectionRemoverFactory); + + // JGit modification: handle unencrypted EC keys + if (basicData == null) { + byte[] nBytes = SXprUtils.readBytes(inputStream, + inputStream.read()); + BigInteger d = new BigInteger(1, nBytes); + SXprUtils.skipCloseParenthesis(inputStream); + return d; + } + + byte[] keyData = basicData[0]; + byte[] protectedAt = basicData[1]; + + // + // parse the secret key S-expr + // + InputStream keyIn = new ByteArrayInputStream(keyData); + + SXprUtils.skipOpenParenthesis(keyIn); + SXprUtils.skipOpenParenthesis(keyIn); + BigInteger d = readBigInteger("d", keyIn); + SXprUtils.skipCloseParenthesis(keyIn); + + // JGit modification: OCB-encrypted keys don't have and don't need a + // hash + if (keyProtectionRemoverFactory instanceof OCBPBEProtectionRemoverFactory) { + return d; + } + + SXprUtils.skipOpenParenthesis(keyIn); + + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("hash")) { + throw new PGPException("hash keyword expected"); + } + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("sha1")) { + throw new PGPException("hash keyword expected"); + } + + byte[] hashBytes = SXprUtils.readBytes(keyIn, keyIn.read()); + + SXprUtils.skipCloseParenthesis(keyIn); + + if (digestProvider != null) { + PGPDigestCalculator digestCalculator = digestProvider + .get(HashAlgorithmTags.SHA1); + + OutputStream dOut = digestCalculator.getOutputStream(); + + dOut.write(Strings.toByteArray("(3:ecc")); + + dOut.write(Strings.toByteArray("(" + curveID.length() + ":" + + curveID + curveName.length() + ":" + curveName + ")")); + + writeCanonical(dOut, "q", qVal); + writeCanonical(dOut, "d", d); + + // check protected-at + if (protectedAt != null) { + dOut.write(protectedAt); + } + + dOut.write(Strings.toByteArray(")")); + + byte[] check = digestCalculator.getDigest(); + + if (!Arrays.constantTimeAreEqual(check, hashBytes)) { + throw new PGPException( + "checksum on protected data failed in SExpr"); + } + } + + return d; + } + + private BigInteger[] processRSASecretKey(InputStream inputStream, + BigInteger n, BigInteger e, + PBEProtectionRemoverFactory keyProtectionRemoverFactory) + throws IOException, PGPException { + String type; + byte[][] basicData = extractData(inputStream, + keyProtectionRemoverFactory); + + byte[] keyData; + byte[] protectedAt = null; + + InputStream keyIn; + BigInteger d; + + if (basicData == null) { + keyIn = inputStream; + byte[] nBytes = SXprUtils.readBytes(inputStream, + inputStream.read()); + d = new BigInteger(1, nBytes); + + SXprUtils.skipCloseParenthesis(inputStream); + + } else { + keyData = basicData[0]; + protectedAt = basicData[1]; + + keyIn = new ByteArrayInputStream(keyData); + + SXprUtils.skipOpenParenthesis(keyIn); + SXprUtils.skipOpenParenthesis(keyIn); + d = readBigInteger("d", keyIn); + } + + // + // parse the secret key S-expr + // + + BigInteger p = readBigInteger("p", keyIn); + BigInteger q = readBigInteger("q", keyIn); + BigInteger u = readBigInteger("u", keyIn); + + // JGit modification: OCB-encrypted keys don't have and don't need a + // hash + if (basicData == null + || keyProtectionRemoverFactory instanceof OCBPBEProtectionRemoverFactory) { + return new BigInteger[] { d, p, q, u }; + } + + SXprUtils.skipCloseParenthesis(keyIn); + + SXprUtils.skipOpenParenthesis(keyIn); + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("hash")) { + throw new PGPException("hash keyword expected"); + } + type = SXprUtils.readString(keyIn, keyIn.read()); + + if (!type.equals("sha1")) { + throw new PGPException("hash keyword expected"); + } + + byte[] hashBytes = SXprUtils.readBytes(keyIn, keyIn.read()); + + SXprUtils.skipCloseParenthesis(keyIn); + + if (digestProvider != null) { + PGPDigestCalculator digestCalculator = digestProvider + .get(HashAlgorithmTags.SHA1); + + OutputStream dOut = digestCalculator.getOutputStream(); + + dOut.write(Strings.toByteArray("(3:rsa")); + + writeCanonical(dOut, "n", n); + writeCanonical(dOut, "e", e); + writeCanonical(dOut, "d", d); + writeCanonical(dOut, "p", p); + writeCanonical(dOut, "q", q); + writeCanonical(dOut, "u", u); + + // check protected-at + if (protectedAt != null) { + dOut.write(protectedAt); + } + + dOut.write(Strings.toByteArray(")")); + + byte[] check = digestCalculator.getDigest(); + + if (!Arrays.constantTimeAreEqual(check, hashBytes)) { + throw new PGPException( + "checksum on protected data failed in SExpr"); + } + } + + return new BigInteger[] { d, p, q, u }; + } + + private void writeCanonical(OutputStream dOut, String label, BigInteger i) + throws IOException { + writeCanonical(dOut, label, i.toByteArray()); + } + + private void writeCanonical(OutputStream dOut, String label, byte[] data) + throws IOException { + dOut.write(Strings.toByteArray( + "(" + label.length() + ":" + label + data.length + ":")); + dOut.write(data); + dOut.write(Strings.toByteArray(")")); + } +} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SXprUtils.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SXprUtils.java new file mode 100644 index 000000000..220aa285f --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SXprUtils.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2000-2021 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + *

    + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software + * and associated documentation files (the "Software"), to deal in the Software without restriction, + *including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + *

    + *

    + * The above copyright notice and this permission notice shall be included in all copies or substantial + * portions of the Software. + *

    + *

    + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *

    + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +// This class is an unmodified copy from Bouncy Castle; needed because it's package-visible only and used by SExprParser. + +import java.io.IOException; +import java.io.InputStream; + +import org.bouncycastle.bcpg.HashAlgorithmTags; +import org.bouncycastle.bcpg.S2K; +import org.bouncycastle.util.io.Streams; + +/** + * Utility functions for looking a S-expression keys. This class will move when + * it finds a better home! + *

    + * Format documented here: + * http://git.gnupg.org/cgi-bin/gitweb.cgi?p=gnupg.git;a=blob;f=agent/keyformat.txt;h=42c4b1f06faf1bbe71ffadc2fee0fad6bec91a97;hb=refs/heads/master + *

    + */ +class SXprUtils { + private static int readLength(InputStream in, int ch) throws IOException { + int len = ch - '0'; + + while ((ch = in.read()) >= 0 && ch != ':') { + len = len * 10 + ch - '0'; + } + + return len; + } + + static String readString(InputStream in, int ch) throws IOException { + int len = readLength(in, ch); + + char[] chars = new char[len]; + + for (int i = 0; i != chars.length; i++) { + chars[i] = (char) in.read(); + } + + return new String(chars); + } + + static byte[] readBytes(InputStream in, int ch) throws IOException { + int len = readLength(in, ch); + + byte[] data = new byte[len]; + + Streams.readFully(in, data); + + return data; + } + + static S2K parseS2K(InputStream in) throws IOException { + skipOpenParenthesis(in); + + // Algorithm is hard-coded to SHA1 below anyway. + readString(in, in.read()); + byte[] iv = readBytes(in, in.read()); + final long iterationCount = Long.parseLong(readString(in, in.read())); + + skipCloseParenthesis(in); + + // we have to return the actual iteration count provided. + S2K s2k = new S2K(HashAlgorithmTags.SHA1, iv, (int) iterationCount) { + @Override + public long getIterationCount() { + return iterationCount; + } + }; + + return s2k; + } + + static void skipOpenParenthesis(InputStream in) throws IOException { + int ch = in.read(); + if (ch != '(') { + throw new IOException( + "unknown character encountered: " + (char) ch); //$NON-NLS-1$ + } + } + + static void skipCloseParenthesis(InputStream in) throws IOException { + int ch = in.read(); + if (ch != ')') { + throw new IOException("unknown character encountered"); //$NON-NLS-1$ + } + } +} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SecretKeys.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SecretKeys.java new file mode 100644 index 000000000..269a1ba0f --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/keys/SecretKeys.java @@ -0,0 +1,597 @@ +/* + * Copyright (C) 2021 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal.keys; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.StreamCorruptedException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Arrays; + +import org.bouncycastle.openpgp.PGPException; +import org.bouncycastle.openpgp.PGPPublicKey; +import org.bouncycastle.openpgp.PGPSecretKey; +import org.bouncycastle.openpgp.operator.PBEProtectionRemoverFactory; +import org.bouncycastle.openpgp.operator.PGPDigestCalculatorProvider; +import org.bouncycastle.openpgp.operator.jcajce.JcePBEProtectionRemoverFactory; +import org.bouncycastle.util.io.Streams; +import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.errors.UnsupportedCredentialItem; +import org.eclipse.jgit.gpg.bc.internal.BCText; +import org.eclipse.jgit.util.RawParseUtils; + +/** + * Utilities for reading GPG secret keys from a gpg-agent key file. + */ +public final class SecretKeys { + + private SecretKeys() { + // No instantiation. + } + + /** + * Something that can supply a passphrase to decrypt an encrypted secret + * key. + */ + public interface PassphraseSupplier { + + /** + * Supplies a passphrase. + * + * @return the passphrase + * @throws PGPException + * if no passphrase can be obtained + * @throws CanceledException + * if the user canceled passphrase entry + * @throws UnsupportedCredentialItem + * if an internal error occurred + * @throws URISyntaxException + * if an internal error occurred + */ + char[] getPassphrase() throws PGPException, CanceledException, + UnsupportedCredentialItem, URISyntaxException; + } + + private static final byte[] PROTECTED_KEY = "protected-private-key" //$NON-NLS-1$ + .getBytes(StandardCharsets.US_ASCII); + + private static final byte[] OCB_PROTECTED = "openpgp-s2k3-ocb-aes" //$NON-NLS-1$ + .getBytes(StandardCharsets.US_ASCII); + + /** + * Reads a GPG secret key from the given stream. + * + * @param in + * {@link InputStream} to read from, doesn't need to be buffered + * @param calculatorProvider + * for checking digests + * @param passphraseSupplier + * for decrypting encrypted keys + * @param publicKey + * the secret key should be for + * @return the secret key + * @throws IOException + * if the stream cannot be parsed + * @throws PGPException + * if thrown by the underlying S-Expression parser, for instance + * when the passphrase is wrong + * @throws CanceledException + * if thrown by the {@code passphraseSupplier} + * @throws UnsupportedCredentialItem + * if thrown by the {@code passphraseSupplier} + * @throws URISyntaxException + * if thrown by the {@code passphraseSupplier} + */ + public static PGPSecretKey readSecretKey(InputStream in, + PGPDigestCalculatorProvider calculatorProvider, + PassphraseSupplier passphraseSupplier, PGPPublicKey publicKey) + throws IOException, PGPException, CanceledException, + UnsupportedCredentialItem, URISyntaxException { + byte[] data = Streams.readAll(in); + if (data.length == 0) { + throw new EOFException(); + } else if (data.length < 4 + PROTECTED_KEY.length) { + // +4 for "(21:" for a binary protected key + throw new IOException( + MessageFormat.format(BCText.get().secretKeyTooShort, + Integer.toUnsignedString(data.length))); + } + SExprParser parser = new SExprParser(calculatorProvider); + byte firstChar = data[0]; + try { + if (firstChar == '(') { + // Binary format. + PBEProtectionRemoverFactory decryptor = null; + if (matches(data, 4, PROTECTED_KEY)) { + // AES/CBC encrypted. + decryptor = new JcePBEProtectionRemoverFactory( + passphraseSupplier.getPassphrase(), + calculatorProvider); + } + try (InputStream sIn = new ByteArrayInputStream(data)) { + return parser.parseSecretKey(sIn, decryptor, publicKey); + } + } + // Assume it's the new key-value format. + try (ByteArrayInputStream keyIn = new ByteArrayInputStream(data)) { + byte[] rawData = keyFromNameValueFormat(keyIn); + if (!matches(rawData, 1, PROTECTED_KEY)) { + // Not encrypted human-readable format. + try (InputStream sIn = new ByteArrayInputStream( + convertSexpression(rawData))) { + return parser.parseSecretKey(sIn, null, publicKey); + } + } + // An encrypted key from a key-value file. Most likely AES/OCB + // encrypted. + boolean isOCB[] = { false }; + byte[] sExp = convertSexpression(rawData, isOCB); + PBEProtectionRemoverFactory decryptor; + if (isOCB[0]) { + decryptor = new OCBPBEProtectionRemoverFactory( + passphraseSupplier.getPassphrase(), + calculatorProvider, getAad(sExp)); + } else { + decryptor = new JcePBEProtectionRemoverFactory( + passphraseSupplier.getPassphrase(), + calculatorProvider); + } + try (InputStream sIn = new ByteArrayInputStream(sExp)) { + return parser.parseSecretKey(sIn, decryptor, publicKey); + } + } + } catch (IOException e) { + throw new PGPException(e.getLocalizedMessage(), e); + } + } + + /** + * Extract the AAD for the OCB decryption from an s-expression. + * + * @param sExp + * buffer containing a valid binary s-expression + * @return the AAD + */ + private static byte[] getAad(byte[] sExp) { + // Given a key + // @formatter:off + // (protected-private-key (rsa ... (protected openpgp-s2k3-ocb-aes ... )(protected-at ...))) + // A B C D + // The AAD is [A..B)[C..D). (From the binary serialized form.) + // @formatter:on + int i = 1; // Skip initial '(' + while (sExp[i] != '(') { + i++; + } + int aadStart = i++; + int aadEnd = skip(sExp, aadStart); + byte[] protectedPrefix = "(9:protected" //$NON-NLS-1$ + .getBytes(StandardCharsets.US_ASCII); + while (!matches(sExp, i, protectedPrefix)) { + i++; + } + int protectedStart = i; + int protectedEnd = skip(sExp, protectedStart); + byte[] aadData = new byte[aadEnd - aadStart + - (protectedEnd - protectedStart)]; + System.arraycopy(sExp, aadStart, aadData, 0, protectedStart - aadStart); + System.arraycopy(sExp, protectedEnd, aadData, protectedStart - aadStart, + aadEnd - protectedEnd); + return aadData; + } + + /** + * Skips a list including nested lists. + * + * @param sExp + * buffer containing valid binary s-expression data + * @param start + * index of the opening '(' of the list to skip + * @return the index after the closing ')' of the skipped list + */ + private static int skip(byte[] sExp, int start) { + int i = start + 1; + int depth = 1; + while (depth > 0) { + switch (sExp[i]) { + case '(': + depth++; + break; + case ')': + depth--; + break; + default: + // We must be on a length + int j = i; + while (sExp[j] >= '0' && sExp[j] <= '9') { + j++; + } + // j is on the colon + int length = Integer.parseInt( + new String(sExp, i, j - i, StandardCharsets.US_ASCII)); + i = j + length; + } + i++; + } + return i; + } + + /** + * Checks whether the {@code needle} matches {@code src} at offset + * {@code from}. + * + * @param src + * to match against {@code needle} + * @param from + * position in {@code src} to start matching + * @param needle + * to match against + * @return {@code true} if {@code src} contains {@code needle} at position + * {@code from}, {@code false} otherwise + */ + private static boolean matches(byte[] src, int from, byte[] needle) { + if (from < 0 || from + needle.length > src.length) { + return false; + } + return org.bouncycastle.util.Arrays.constantTimeAreEqual(needle.length, + src, from, needle, 0); + } + + /** + * Converts a human-readable serialized s-expression into a binary + * serialized s-expression. + * + * @param humanForm + * to convert + * @return the converted s-expression + * @throws IOException + * if the conversion fails + */ + private static byte[] convertSexpression(byte[] humanForm) + throws IOException { + boolean[] isOCB = { false }; + return convertSexpression(humanForm, isOCB); + } + + /** + * Converts a human-readable serialized s-expression into a binary + * serialized s-expression. + * + * @param humanForm + * to convert + * @param isOCB + * returns whether the s-expression specified AES/OCB encryption + * @return the converted s-expression + * @throws IOException + * if the conversion fails + */ + private static byte[] convertSexpression(byte[] humanForm, boolean[] isOCB) + throws IOException { + int pos = 0; + try (ByteArrayOutputStream out = new ByteArrayOutputStream( + humanForm.length)) { + while (pos < humanForm.length) { + byte b = humanForm[pos]; + if (b == '(' || b == ')') { + out.write(b); + pos++; + } else if (isGpgSpace(b)) { + pos++; + } else if (b == '#') { + // Hex value follows up to the next # + int i = ++pos; + while (i < humanForm.length && isHex(humanForm[i])) { + i++; + } + if (i == pos || humanForm[i] != '#') { + throw new StreamCorruptedException( + BCText.get().sexprHexNotClosed); + } + if ((i - pos) % 2 != 0) { + throw new StreamCorruptedException( + BCText.get().sexprHexOdd); + } + int l = (i - pos) / 2; + out.write(Integer.toString(l) + .getBytes(StandardCharsets.US_ASCII)); + out.write(':'); + while (pos < i) { + int x = (nibble(humanForm[pos]) << 4) + | nibble(humanForm[pos + 1]); + pos += 2; + out.write(x); + } + pos = i + 1; + } else if (isTokenChar(b)) { + // Scan the token + int start = pos++; + while (pos < humanForm.length + && isTokenChar(humanForm[pos])) { + pos++; + } + int l = pos - start; + if (pos - start == OCB_PROTECTED.length + && matches(humanForm, start, OCB_PROTECTED)) { + isOCB[0] = true; + } + out.write(Integer.toString(l) + .getBytes(StandardCharsets.US_ASCII)); + out.write(':'); + out.write(humanForm, start, pos - start); + } else if (b == '"') { + // Potentially quoted string. + int start = ++pos; + boolean escaped = false; + while (pos < humanForm.length + && (escaped || humanForm[pos] != '"')) { + int ch = humanForm[pos++]; + escaped = !escaped && ch == '\\'; + } + if (pos >= humanForm.length) { + throw new StreamCorruptedException( + BCText.get().sexprStringNotClosed); + } + // start is on the first character of the string, pos on the + // closing quote. + byte[] dq = dequote(humanForm, start, pos); + out.write(Integer.toString(dq.length) + .getBytes(StandardCharsets.US_ASCII)); + out.write(':'); + out.write(dq); + pos++; + } else { + throw new StreamCorruptedException( + MessageFormat.format(BCText.get().sexprUnhandled, + Integer.toHexString(b & 0xFF))); + } + } + return out.toByteArray(); + } + } + + /** + * GPG-style string de-quoting, which is basically C-style, with some + * literal CR/LF escaping. + * + * @param in + * buffer containing the quoted string + * @param from + * index after the opening quote in {@code in} + * @param to + * index of the closing quote in {@code in} + * @return the dequoted raw string value + * @throws StreamCorruptedException + */ + private static byte[] dequote(byte[] in, int from, int to) + throws StreamCorruptedException { + // Result must be shorter or have the same length + byte[] out = new byte[to - from]; + int j = 0; + int i = from; + while (i < to) { + byte b = in[i++]; + if (b != '\\') { + out[j++] = b; + continue; + } + if (i == to) { + throw new StreamCorruptedException( + BCText.get().sexprStringInvalidEscapeAtEnd); + } + b = in[i++]; + switch (b) { + case 'b': + out[j++] = '\b'; + break; + case 'f': + out[j++] = '\f'; + break; + case 'n': + out[j++] = '\n'; + break; + case 'r': + out[j++] = '\r'; + break; + case 't': + out[j++] = '\t'; + break; + case 'v': + out[j++] = 0x0B; + break; + case '"': + case '\'': + case '\\': + out[j++] = b; + break; + case '\r': + // Escaped literal line end. If an LF is following, skip that, + // too. + if (i < to && in[i] == '\n') { + i++; + } + break; + case '\n': + // Same for LF possibly followed by CR. + if (i < to && in[i] == '\r') { + i++; + } + break; + case 'x': + if (i + 1 >= to || !isHex(in[i]) || !isHex(in[i + 1])) { + throw new StreamCorruptedException( + BCText.get().sexprStringInvalidHexEscape); + } + out[j++] = (byte) ((nibble(in[i]) << 4) | nibble(in[i + 1])); + i += 2; + break; + case '0': + case '1': + case '2': + case '3': + if (i + 2 >= to || !isOctal(in[i]) || !isOctal(in[i + 1]) + || !isOctal(in[i + 2])) { + throw new StreamCorruptedException( + BCText.get().sexprStringInvalidOctalEscape); + } + out[j++] = (byte) (((((in[i] - '0') << 3) + | (in[i + 1] - '0')) << 3) | (in[i + 2] - '0')); + i += 3; + break; + default: + throw new StreamCorruptedException(MessageFormat.format( + BCText.get().sexprStringInvalidEscape, + Integer.toHexString(b & 0xFF))); + } + } + return Arrays.copyOf(out, j); + } + + /** + * Extracts the key from a GPG name-value-pair key file. + *

    + * Package-visible for tests only. + *

    + * + * @param in + * {@link InputStream} to read from; should be buffered + * @return the raw key data as extracted from the file + * @throws IOException + * if the {@code in} stream cannot be read or does not contain a + * key + */ + static byte[] keyFromNameValueFormat(InputStream in) throws IOException { + // It would be nice if we could use RawParseUtils here, but GPG compares + // names case-insensitively. We're only interested in the "Key:" + // name-value pair. + int[] nameLow = { 'k', 'e', 'y', ':' }; + int[] nameCap = { 'K', 'E', 'Y', ':' }; + int nameIdx = 0; + for (;;) { + int next = in.read(); + if (next < 0) { + throw new EOFException(); + } + if (next == '\n') { + nameIdx = 0; + } else if (nameIdx >= 0) { + if (nameLow[nameIdx] == next || nameCap[nameIdx] == next) { + nameIdx++; + if (nameIdx == nameLow.length) { + break; + } + } else { + nameIdx = -1; + } + } + } + // We're after "Key:". Read the value as continuation lines. + int last = ':'; + byte[] rawData; + try (ByteArrayOutputStream out = new ByteArrayOutputStream(8192)) { + for (;;) { + int next = in.read(); + if (next < 0) { + break; + } + if (last == '\n') { + if (next == ' ' || next == '\t') { + // Continuation line; skip this whitespace + last = next; + continue; + } + break; // Not a continuation line + } + out.write(next); + last = next; + } + rawData = out.toByteArray(); + } + // GPG trims off trailing whitespace, and a line having only whitespace + // is a single LF. + try (ByteArrayOutputStream out = new ByteArrayOutputStream( + rawData.length)) { + int lineStart = 0; + boolean trimLeading = true; + while (lineStart < rawData.length) { + int nextLineStart = RawParseUtils.nextLF(rawData, lineStart); + if (trimLeading) { + while (lineStart < nextLineStart + && isGpgSpace(rawData[lineStart])) { + lineStart++; + } + } + // Trim trailing + int i = nextLineStart - 1; + while (lineStart < i && isGpgSpace(rawData[i])) { + i--; + } + if (i <= lineStart) { + // Empty line signifies LF + out.write('\n'); + trimLeading = true; + } else { + out.write(rawData, lineStart, i - lineStart + 1); + trimLeading = false; + } + lineStart = nextLineStart; + } + return out.toByteArray(); + } + } + + private static boolean isGpgSpace(int ch) { + return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n'; + } + + private static boolean isTokenChar(int ch) { + switch (ch) { + case '-': + case '.': + case '/': + case '_': + case ':': + case '*': + case '+': + case '=': + return true; + default: + if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') + || (ch >= '0' && ch <= '9')) { + return true; + } + return false; + } + } + + private static boolean isHex(int ch) { + return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') + || (ch >= 'a' && ch <= 'f'); + } + + private static boolean isOctal(int ch) { + return (ch >= '0' && ch <= '7'); + } + + private static int nibble(int ch) { + if (ch >= '0' && ch <= '9') { + return ch - '0'; + } else if (ch >= 'A' && ch <= 'F') { + return ch - 'A' + 10; + } else if (ch >= 'a' && ch <= 'f') { + return ch - 'a' + 10; + } + return -1; + } +} diff --git a/org.eclipse.jgit.http.apache/META-INF/MANIFEST.MF b/org.eclipse.jgit.http.apache/META-INF/MANIFEST.MF index 0fef0cfbd..0f2d7a3b6 100644 --- a/org.eclipse.jgit.http.apache/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.http.apache/META-INF/MANIFEST.MF @@ -9,20 +9,20 @@ Bundle-Localization: plugin Bundle-Vendor: %Bundle-Vendor Bundle-ActivationPolicy: lazy Import-Package: org.apache.http;version="[4.3.0,5.0.0)", - org.apache.http.client;version="[4.3.0,5.0.0)", - org.apache.http.client.config;version="[4.3.0,5.0.0)", - org.apache.http.client.methods;version="[4.3.0,5.0.0)", - org.apache.http.client.params;version="[4.3.0,5.0.0)", + org.apache.http.client;version="[4.4.0,5.0.0)", + org.apache.http.client.config;version="[4.4.0,5.0.0)", + org.apache.http.client.methods;version="[4.4.0,5.0.0)", + org.apache.http.client.params;version="[4.4.0,5.0.0)", org.apache.http.config;version="[4.3.0,5.0.0)", - org.apache.http.conn;version="[4.3.0,5.0.0)", - org.apache.http.conn.params;version="[4.3.0,5.0.0)", - org.apache.http.conn.scheme;version="[4.3.0,5.0.0)", - org.apache.http.conn.socket;version="[4.3.0,5.0.0)", - org.apache.http.conn.ssl;version="[4.3.0,5.0.0)", - org.apache.http.conn.util;version="[4.3.0,5.0.0)", + org.apache.http.conn;version="[4.4.0,5.0.0)", + org.apache.http.conn.params;version="[4.4.0,5.0.0)", + org.apache.http.conn.scheme;version="[4.4.0,5.0.0)", + org.apache.http.conn.socket;version="[4.4.0,5.0.0)", + org.apache.http.conn.ssl;version="[4.4.0,5.0.0)", + org.apache.http.conn.util;version="[4.4.0,5.0.0)", org.apache.http.entity;version="[4.3.0,5.0.0)", - org.apache.http.impl.client;version="[4.3.0,5.0.0)", - org.apache.http.impl.conn;version="[4.3.0,5.0.0)", + org.apache.http.impl.client;version="[4.4.0,5.0.0)", + org.apache.http.impl.conn;version="[4.4.0,5.0.0)", org.apache.http.params;version="[4.3.0,5.0.0)", org.apache.http.ssl;version="[4.3.0,5.0.0)", org.eclipse.jgit.annotations;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.http.apache/resources/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.properties b/org.eclipse.jgit.http.apache/resources/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.properties index d2e521698..b7b9af0a4 100644 --- a/org.eclipse.jgit.http.apache/resources/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.properties +++ b/org.eclipse.jgit.http.apache/resources/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.properties @@ -1 +1,2 @@ +httpWrongConnectionType=Wrong connection type: expected {0}, got {1}. unexpectedSSLContextException=unexpected exception when searching for the TLS protocol diff --git a/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnection.java b/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnection.java index ed05f0a8d..90348f54b 100644 --- a/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnection.java +++ b/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnection.java @@ -57,9 +57,7 @@ import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.util.PublicSuffixMatcherLoader; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.SystemDefaultCredentialsProvider; @@ -103,7 +101,11 @@ public class HttpClientConnection implements HttpConnection { private HostnameVerifier hostnameverifier; - SSLContext ctx; + private SSLContext ctx; + + private SSLConnectionSocketFactory socketFactory; + + private boolean usePooling = true; private HttpClient getClient() { if (client == null) { @@ -125,11 +127,18 @@ private HttpClient getClient() { configBuilder .setRedirectsEnabled(followRedirects.booleanValue()); } - SSLConnectionSocketFactory sslConnectionFactory = getSSLSocketFactory(); + boolean pooled = true; + SSLConnectionSocketFactory sslConnectionFactory; + if (socketFactory != null) { + pooled = usePooling; + sslConnectionFactory = socketFactory; + } else { + // Legacy implementation. + pooled = (hostnameverifier == null); + sslConnectionFactory = getSSLSocketFactory(); + } clientBuilder.setSSLSocketFactory(sslConnectionFactory); - if (hostnameverifier != null) { - // Using a custom verifier: we don't want pooled connections - // with this. + if (!pooled) { Registry registry = RegistryBuilder . create() .register("https", sslConnectionFactory) @@ -147,14 +156,19 @@ private HttpClient getClient() { return client; } + void setSSLSocketFactory(@NonNull SSLConnectionSocketFactory factory, + boolean isDefault) { + socketFactory = factory; + usePooling = isDefault; + } + private SSLConnectionSocketFactory getSSLSocketFactory() { HostnameVerifier verifier = hostnameverifier; SSLContext context; if (verifier == null) { // Use defaults - context = SSLContexts.createDefault(); - verifier = new DefaultHostnameVerifier( - PublicSuffixMatcherLoader.getDefault()); + context = SSLContexts.createSystemDefault(); + verifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier(); } else { // Using a custom verifier. Attention: configure() must have been // called already, otherwise one gets a "context not initialized" diff --git a/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnectionFactory.java b/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnectionFactory.java index 3c05cdef8..4de3e470f 100644 --- a/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnectionFactory.java +++ b/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/HttpClientConnectionFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Christian Halstrick and others + * Copyright (C) 2013, 2020 Christian Halstrick and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,27 +12,100 @@ import java.io.IOException; import java.net.Proxy; import java.net.URL; +import java.security.GeneralSecurityException; +import java.text.MessageFormat; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.TrustManager; + +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.eclipse.jgit.transport.http.HttpConnection; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; +import org.eclipse.jgit.transport.http.HttpConnectionFactory2; +import org.eclipse.jgit.transport.http.NoCheckX509TrustManager; +import org.eclipse.jgit.transport.http.apache.internal.HttpApacheText; +import org.eclipse.jgit.util.HttpSupport; /** - * A factory returning instances of - * {@link org.eclipse.jgit.transport.http.apache.HttpClientConnection} + * A factory returning instances of {@link HttpClientConnection}. * * @since 3.3 */ -public class HttpClientConnectionFactory implements HttpConnectionFactory { - /** {@inheritDoc} */ +public class HttpClientConnectionFactory implements HttpConnectionFactory2 { + @Override public HttpConnection create(URL url) throws IOException { return new HttpClientConnection(url.toString()); } - /** {@inheritDoc} */ @Override - public HttpConnection create(URL url, Proxy proxy) - throws IOException { + public HttpConnection create(URL url, Proxy proxy) throws IOException { return new HttpClientConnection(url.toString(), proxy); } + + @Override + public GitSession newSession() { + return new HttpClientSession(); + } + + private static class HttpClientSession implements GitSession { + + private SSLContext securityContext; + + private SSLConnectionSocketFactory socketFactory; + + private boolean isDefault; + + @Override + public HttpClientConnection configure(HttpConnection connection, + boolean sslVerify) + throws IOException, GeneralSecurityException { + if (!(connection instanceof HttpClientConnection)) { + throw new IllegalArgumentException(MessageFormat.format( + HttpApacheText.get().httpWrongConnectionType, + HttpClientConnection.class.getName(), + connection.getClass().getName())); + } + HttpClientConnection conn = (HttpClientConnection) connection; + String scheme = conn.getURL().getProtocol(); + if (!"https".equals(scheme)) { //$NON-NLS-1$ + return conn; + } + if (securityContext == null || isDefault != sslVerify) { + isDefault = sslVerify; + HostnameVerifier verifier; + if (sslVerify) { + securityContext = SSLContext.getDefault(); + verifier = SSLConnectionSocketFactory + .getDefaultHostnameVerifier(); + } else { + securityContext = SSLContext.getInstance("TLS"); + TrustManager[] trustAllCerts = { + new NoCheckX509TrustManager() }; + securityContext.init(null, trustAllCerts, null); + verifier = (name, session) -> true; + } + socketFactory = new SSLConnectionSocketFactory(securityContext, + verifier) { + + @Override + protected void prepareSocket(SSLSocket socket) + throws IOException { + super.prepareSocket(socket); + HttpSupport.configureTLS(socket); + } + }; + } + conn.setSSLSocketFactory(socketFactory, isDefault); + return conn; + } + + @Override + public void close() { + securityContext = null; + socketFactory = null; + } + + } } diff --git a/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.java b/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.java index 907ab98cc..677d7d792 100644 --- a/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.java +++ b/org.eclipse.jgit.http.apache/src/org/eclipse/jgit/transport/http/apache/internal/HttpApacheText.java @@ -27,5 +27,6 @@ public static HttpApacheText get() { } // @formatter:off + /***/ public String httpWrongConnectionType; /***/ public String unexpectedSSLContextException; } diff --git a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java index c3d72552a..e90580b75 100644 --- a/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java +++ b/org.eclipse.jgit.http.server/src/org/eclipse/jgit/http/server/InfoPacksServlet.java @@ -20,7 +20,7 @@ import javax.servlet.http.HttpServletResponse; import org.eclipse.jgit.internal.storage.file.ObjectDirectory; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.lib.ObjectDatabase; /** Sends the current list of pack files, sorted most recent first. */ @@ -38,7 +38,7 @@ private static String packList(HttpServletRequest req) { final StringBuilder out = new StringBuilder(); final ObjectDatabase db = getRepository(req).getObjectDatabase(); if (db instanceof ObjectDirectory) { - for (PackFile pack : ((ObjectDirectory) db).getPacks()) { + for (Pack pack : ((ObjectDirectory) db).getPacks()) { out.append("P "); out.append(pack.getPackFile().getName()); out.append('\n'); diff --git a/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..b853c6a7e 100644 --- a/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.http.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.http.test/META-INF/MANIFEST.MF index a63ebdaeb..c085e7eff 100644 --- a/org.eclipse.jgit.http.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.http.test/META-INF/MANIFEST.MF @@ -12,7 +12,7 @@ Import-Package: javax.servlet;version="[2.5.0,3.2.0)", org.apache.commons.codec;version="[1.6.0,2.0.0)", org.apache.commons.codec.binary;version="[1.6.0,2.0.0)", org.apache.http;version="[4.3.0,5.0.0)", - org.apache.http.client;version="[4.3.0,5.0.0)", + org.apache.http.client;version="[4.4.0,5.0.0)", org.apache.http.message;version="[4.3.0,5.0.0)", org.eclipse.jetty.continuation;version="[9.4.5,10.0.0)", org.eclipse.jetty.http;version="[9.4.5,10.0.0)", @@ -28,6 +28,7 @@ Import-Package: javax.servlet;version="[2.5.0,3.2.0)", org.eclipse.jetty.util.log;version="[9.4.5,10.0.0)", org.eclipse.jetty.util.security;version="[9.4.5,10.0.0)", org.eclipse.jetty.util.thread;version="[9.4.5,10.0.0)", + org.eclipse.jgit.api;version="[6.0.0,6.1.0)", org.eclipse.jgit.errors;version="[6.0.0,6.1.0)", org.eclipse.jgit.http.server;version="[6.0.0,6.1.0)", org.eclipse.jgit.http.server.glue;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.http.test/build.properties b/org.eclipse.jgit.http.test/build.properties index e8bacac9a..a909f1301 100644 --- a/org.eclipse.jgit.http.test/build.properties +++ b/org.eclipse.jgit.http.test/build.properties @@ -4,3 +4,5 @@ output.. = bin/ bin.includes = META-INF/,\ .,\ plugin.properties +additional.bundles = org.apache.log4j,\ + org.slf4j.binding.log4j12 diff --git a/org.eclipse.jgit.http.test/src/org/eclipse/jgit/http/test/RefsUnreadableInMemoryRepository.java b/org.eclipse.jgit.http.test/src/org/eclipse/jgit/http/test/RefsUnreadableInMemoryRepository.java index 80cbe8738..4167b038e 100644 --- a/org.eclipse.jgit.http.test/src/org/eclipse/jgit/http/test/RefsUnreadableInMemoryRepository.java +++ b/org.eclipse.jgit.http.test/src/org/eclipse/jgit/http/test/RefsUnreadableInMemoryRepository.java @@ -83,6 +83,17 @@ public List getRefsByPrefix(String prefix) throws IOException { return super.getRefsByPrefix(prefix); } + /** {@inheritDoc} */ + @Override + public List getRefsByPrefixWithExclusions(String include, Set excludes) + throws IOException { + if (failing) { + throw new IOException("disk failed, no refs found"); + } + + return super.getRefsByPrefixWithExclusions(include, excludes); + } + /** {@inheritDoc} */ @Override public Set getTipsWithSha1(ObjectId id) throws IOException { diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java new file mode 100644 index 000000000..c6931ad5b --- /dev/null +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/AllProtocolsHttpTestCase.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2020, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.http.test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.eclipse.jgit.junit.http.HttpTestCase; +import org.eclipse.jgit.transport.HttpTransport; +import org.eclipse.jgit.transport.http.HttpConnectionFactory; +import org.eclipse.jgit.transport.http.JDKHttpConnectionFactory; +import org.eclipse.jgit.transport.http.apache.HttpClientConnectionFactory; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Abstract test base class for running HTTP-related tests with all connection + * factories provided in JGit and with both protocol V0 and V2. + */ +@Ignore +@RunWith(Parameterized.class) +public abstract class AllProtocolsHttpTestCase extends HttpTestCase { + + protected static class TestParameters { + + public final HttpConnectionFactory factory; + + public final boolean enableProtocolV2; + + public TestParameters(HttpConnectionFactory factory, + boolean enableProtocolV2) { + this.factory = factory; + this.enableProtocolV2 = enableProtocolV2; + } + + @Override + public String toString() { + return factory.toString() + " protocol " + + (enableProtocolV2 ? "V2" : "V0"); + } + } + + @Parameters(name = "{0}") + public static Collection data() { + // run all tests with both connection factories we have + HttpConnectionFactory[] factories = new HttpConnectionFactory[] { + new JDKHttpConnectionFactory() { + + @Override + public String toString() { + return this.getClass().getSuperclass().getName(); + } + }, new HttpClientConnectionFactory() { + + @Override + public String toString() { + return this.getClass().getSuperclass().getName(); + } + } }; + List result = new ArrayList<>(); + for (HttpConnectionFactory factory : factories) { + result.add(new TestParameters(factory, false)); + result.add(new TestParameters(factory, true)); + } + return result; + } + + protected final boolean enableProtocolV2; + + protected AllProtocolsHttpTestCase(TestParameters params) { + HttpTransport.setConnectionFactory(params.factory); + enableProtocolV2 = params.enableProtocolV2; + } + + private static HttpConnectionFactory originalFactory; + + @BeforeClass + public static void saveConnectionFactory() { + originalFactory = HttpTransport.getConnectionFactory(); + } + + @AfterClass + public static void restoreConnectionFactory() { + HttpTransport.setConnectionFactory(originalFactory); + } + +} diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java index 6da5f86b3..8b28c4292 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientDumbServerTest.java @@ -35,6 +35,7 @@ import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.transport.FetchConnection; @@ -77,6 +78,9 @@ public void setUp() throws Exception { remoteRepository = src.getRepository(); remoteURI = toURIish(app, srcGit.getName()); + StoredConfig cfg = remoteRepository.getConfig(); + cfg.setInt("protocol", null, "version", 0); + cfg.save(); A_txt = src.blob("A"); A = src.commit().add("A_txt", A_txt).create(); diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java index ccde1fe55..986b5ca92 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/DumbClientSmartServerTest.java @@ -35,6 +35,7 @@ import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.transport.FetchConnection; @@ -42,11 +43,10 @@ import org.eclipse.jgit.transport.Transport; import org.eclipse.jgit.transport.TransportHttp; import org.eclipse.jgit.transport.URIish; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.junit.Before; import org.junit.Test; -public class DumbClientSmartServerTest extends AllFactoriesHttpTestCase { +public class DumbClientSmartServerTest extends AllProtocolsHttpTestCase { private Repository remoteRepository; private URIish remoteURI; @@ -55,8 +55,8 @@ public class DumbClientSmartServerTest extends AllFactoriesHttpTestCase { private RevCommit A, B; - public DumbClientSmartServerTest(HttpConnectionFactory cf) { - super(cf); + public DumbClientSmartServerTest(TestParameters params) { + super(params); } @Override @@ -76,6 +76,9 @@ public void setUp() throws Exception { remoteRepository = src.getRepository(); remoteURI = toURIish(app, srcName); + StoredConfig cfg = remoteRepository.getConfig(); + cfg.setInt("protocol", null, "version", enableProtocolV2 ? 2 : 0); + cfg.save(); A_txt = src.blob("A"); A = src.commit().add("A_txt", A_txt).create(); diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java index 96657761c..df093c185 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/HttpClientTests.java @@ -324,7 +324,22 @@ public void testListRemoteWithoutLocalRepository() throws Exception { } @Test - public void testHttpClientWantsV2ButServerNotConfigured() throws Exception { + public void testHttpClientWantsV2AndServerNotConfigured() throws Exception { + String url = smartAuthNoneURI.toString() + "/info/refs?service=git-upload-pack"; + HttpConnection c = HttpTransport.getConnectionFactory() + .create(new URL(url)); + c.setRequestMethod("GET"); + c.setRequestProperty("Git-Protocol", "version=2"); + assertEquals(200, c.getResponseCode()); + + PacketLineIn pckIn = new PacketLineIn(c.getInputStream()); + assertThat(pckIn.readString(), is("version 2")); + } + + @Test + public void testHttpServerConfiguredToV0() throws Exception { + remoteRepository.getRepository().getConfig().setInt( + "protocol", null, "version", 0); String url = smartAuthNoneURI.toString() + "/info/refs?service=git-upload-pack"; HttpConnection c = HttpTransport.getConnectionFactory() .create(new URL(url)); @@ -342,9 +357,6 @@ public void testHttpClientWantsV2ButServerNotConfigured() throws Exception { @Test public void testV2HttpFirstResponse() throws Exception { - remoteRepository.getRepository().getConfig().setInt( - "protocol", null, "version", 2); - String url = smartAuthNoneURI.toString() + "/info/refs?service=git-upload-pack"; HttpConnection c = HttpTransport.getConnectionFactory() .create(new URL(url)); @@ -364,9 +376,6 @@ public void testV2HttpFirstResponse() throws Exception { @Test public void testV2HttpSubsequentResponse() throws Exception { - remoteRepository.getRepository().getConfig().setInt( - "protocol", null, "version", 2); - String url = smartAuthNoneURI.toString() + "/git-upload-pack"; HttpConnection c = HttpTransport.getConnectionFactory() .create(new URL(url)); diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java index 597fb2e50..7bc50cad8 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerSslTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Thomas Wolf and others + * Copyright (C) 2017, 2020 Thomas Wolf and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -41,6 +41,7 @@ import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.transport.CredentialItem; @@ -48,7 +49,6 @@ import org.eclipse.jgit.transport.Transport; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; -import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.eclipse.jgit.util.HttpSupport; import org.junit.Before; import org.junit.Test; @@ -56,7 +56,7 @@ import org.junit.runners.Parameterized; @RunWith(Parameterized.class) -public class SmartClientSmartServerSslTest extends AllFactoriesHttpTestCase { +public class SmartClientSmartServerSslTest extends AllProtocolsHttpTestCase { // We run these tests with a server on localhost with a self-signed // certificate. We don't do authentication tests here, so there's no need @@ -112,8 +112,8 @@ public boolean get(URIish uri, CredentialItem... items) private RevCommit A, B; - public SmartClientSmartServerSslTest(HttpConnectionFactory cf) { - super(cf); + public SmartClientSmartServerSslTest(TestParameters params) { + super(params); } @Override @@ -128,10 +128,11 @@ public void setUp() throws Exception { final TestRepository src = createTestRepository(); final String srcName = src.getRepository().getDirectory().getName(); - src.getRepository() - .getConfig() - .setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, true); + StoredConfig cfg = src.getRepository().getConfig(); + cfg.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, true); + cfg.setInt("protocol", null, "version", enableProtocolV2 ? 2 : 0); + cfg.save(); GitServlet gs = new GitServlet(); @@ -238,7 +239,7 @@ public void testInitialClone_ViaHttps() throws Exception { fsck(dst, B); List requests = getRequests(); - assertEquals(2, requests.size()); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); } @Test @@ -256,7 +257,7 @@ public void testInitialClone_RedirectToHttps() throws Exception { fsck(dst, B); List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); } @Test diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java index 8d1870a87..887e970a0 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2017 Google Inc. and others + * Copyright (C) 2010, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -22,10 +22,17 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; +import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.io.Writer; +import java.net.Proxy; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.Collections; import java.util.EnumSet; @@ -48,6 +55,8 @@ import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.TransportConfigCallback; import org.eclipse.jgit.errors.RemoteRepositoryException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.errors.UnsupportedCredentialItem; @@ -70,6 +79,7 @@ import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.lib.TextProgressMonitor; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; @@ -86,13 +96,14 @@ import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UploadPack; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.http.HttpConnection; import org.eclipse.jgit.transport.http.HttpConnectionFactory; import org.eclipse.jgit.util.HttpSupport; import org.eclipse.jgit.util.SystemReader; import org.junit.Before; import org.junit.Test; -public class SmartClientSmartServerTest extends AllFactoriesHttpTestCase { +public class SmartClientSmartServerTest extends AllProtocolsHttpTestCase { private static final String HDR_TRANSFER_ENCODING = "Transfer-Encoding"; private AdvertiseRefsHook advertiseRefsHook; @@ -120,8 +131,8 @@ public class SmartClientSmartServerTest extends AllFactoriesHttpTestCase { private RevCommit A, B, unreachableCommit; - public SmartClientSmartServerTest(HttpConnectionFactory cf) { - super(cf); + public SmartClientSmartServerTest(TestParameters params) { + super(params); } @Override @@ -131,10 +142,11 @@ public void setUp() throws Exception { final TestRepository src = createTestRepository(); final String srcName = src.getRepository().getDirectory().getName(); - src.getRepository() - .getConfig() - .setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, true); + StoredConfig cfg = src.getRepository().getConfig(); + cfg.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, true); + cfg.setInt("protocol", null, "version", enableProtocolV2 ? 2 : 0); + cfg.save(); GitServlet gs = new GitServlet(); gs.setUploadPackFactory((HttpServletRequest req, Repository db) -> { @@ -448,7 +460,7 @@ public void testListRemote() throws IOException { assertEquals(B, map.get(Constants.HEAD).getObjectId()); List requests = getRequests(); - assertEquals(1, requests.size()); + assertEquals(enableProtocolV2 ? 2 : 1, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -458,7 +470,22 @@ public void testListRemote() throws IOException { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info .getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(1); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -576,9 +603,10 @@ public void testInitialClone_Small() throws Exception { } List requests = getRequests(); - assertEquals(2, requests.size()); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); - AccessEvent info = requests.get(0); + int requestNumber = 0; + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(remoteURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -586,9 +614,24 @@ public void testInitialClone_Small() throws Exception { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info .getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } - AccessEvent service = requests.get(1); + AccessEvent service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -602,6 +645,63 @@ public void testInitialClone_Small() throws Exception { .getResponseHeader(HDR_CONTENT_TYPE)); } + @Test + public void test_CloneWithCustomFactory() throws Exception { + HttpConnectionFactory globalFactory = HttpTransport + .getConnectionFactory(); + HttpConnectionFactory failingConnectionFactory = new HttpConnectionFactory() { + + @Override + public HttpConnection create(URL url) throws IOException { + throw new IOException("Should not be reached"); + } + + @Override + public HttpConnection create(URL url, Proxy proxy) + throws IOException { + throw new IOException("Should not be reached"); + } + }; + HttpTransport.setConnectionFactory(failingConnectionFactory); + try { + File tmp = createTempDirectory("cloneViaApi"); + boolean[] localFactoryUsed = { false }; + TransportConfigCallback callback = new TransportConfigCallback() { + + @Override + public void configure(Transport transport) { + if (transport instanceof TransportHttp) { + ((TransportHttp) transport).setHttpConnectionFactory( + new HttpConnectionFactory() { + + @Override + public HttpConnection create(URL url) + throws IOException { + localFactoryUsed[0] = true; + return globalFactory.create(url); + } + + @Override + public HttpConnection create(URL url, + Proxy proxy) throws IOException { + localFactoryUsed[0] = true; + return globalFactory.create(url, proxy); + } + }); + } + } + }; + try (Git git = Git.cloneRepository().setDirectory(tmp) + .setTransportConfigCallback(callback) + .setURI(remoteURI.toPrivateString()).call()) { + assertTrue("Should have used the local HttpConnectionFactory", + localFactoryUsed[0]); + } + } finally { + HttpTransport.setConnectionFactory(globalFactory); + } + } + private void initialClone_Redirect(int nofRedirects, int code) throws Exception { initialClone_Redirect(nofRedirects, code, false); @@ -628,7 +728,8 @@ private void initialClone_Redirect(int nofRedirects, int code, } List requests = getRequests(); - assertEquals(2 + nofRedirects, requests.size()); + assertEquals((enableProtocolV2 ? 3 : 2) + nofRedirects, + requests.size()); int n = 0; while (n < nofRedirects) { @@ -644,7 +745,22 @@ private void initialClone_Redirect(int nofRedirects, int code, assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(n++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } AccessEvent service = requests.get(n++); assertEquals("POST", service.getMethod()); @@ -756,7 +872,7 @@ public void testInitialClone_RedirectOnPostAllowed() throws Exception { } List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -766,24 +882,27 @@ public void testInitialClone_RedirectOnPostAllowed() throws Exception { assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } AccessEvent redirect = requests.get(1); assertEquals("POST", redirect.getMethod()); assertEquals(301, redirect.getStatus()); - AccessEvent service = requests.get(2); - assertEquals("POST", service.getMethod()); - assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); - assertEquals(0, service.getParameters().size()); - assertNotNull("has content-length", - service.getRequestHeader(HDR_CONTENT_LENGTH)); - assertNull("not chunked", - service.getRequestHeader(HDR_TRANSFER_ENCODING)); - - assertEquals(200, service.getStatus()); - assertEquals("application/x-git-upload-pack-result", - service.getResponseHeader(HDR_CONTENT_TYPE)); + for (int i = 2; i < requests.size(); i++) { + AccessEvent service = requests.get(i); + assertEquals("POST", service.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); + assertEquals(0, service.getParameters().size()); + assertNotNull("has content-length", + service.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + service.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals(200, service.getStatus()); + assertEquals("application/x-git-upload-pack-result", + service.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -817,6 +936,35 @@ public void testInitialClone_RedirectForbidden() throws Exception { } } + private void assertFetchRequests(List requests, int index) { + AccessEvent info = requests.get(index++); + assertEquals("GET", info.getMethod()); + assertEquals(join(authURI, "info/refs"), info.getPath()); + assertEquals(1, info.getParameters().size()); + assertEquals("git-upload-pack", info.getParameter("service")); + assertEquals(200, info.getStatus()); + assertEquals("application/x-git-upload-pack-advertisement", + info.getResponseHeader(HDR_CONTENT_TYPE)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } + + for (int i = index; i < requests.size(); i++) { + AccessEvent service = requests.get(i); + assertEquals("POST", service.getMethod()); + assertEquals(join(authURI, "git-upload-pack"), service.getPath()); + assertEquals(0, service.getParameters().size()); + assertNotNull("has content-length", + service.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + service.getRequestHeader(HDR_TRANSFER_ENCODING)); + + assertEquals(200, service.getStatus()); + assertEquals("application/x-git-upload-pack-result", + service.getResponseHeader(HDR_CONTENT_TYPE)); + } + } + @Test public void testInitialClone_WithAuthentication() throws Exception { try (Repository dst = createBareRepository(); @@ -830,34 +978,167 @@ public void testInitialClone_WithAuthentication() throws Exception { } List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); assertEquals(401, info.getStatus()); - info = requests.get(1); + assertFetchRequests(requests, 1); + } + + @Test + public void testInitialClone_WithPreAuthentication() throws Exception { + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, authURI)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password); + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + assertTrue(dst.getObjectDatabase().has(A_txt)); + assertEquals(B, dst.exactRef(master).getObjectId()); + fsck(dst, B); + } + + List requests = getRequests(); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); + + assertFetchRequests(requests, 0); + } + + @Test + public void testInitialClone_WithPreAuthenticationCleared() + throws Exception { + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, authURI)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password); + ((TransportHttp) t).setPreemptiveBasicAuthentication(null, null); + t.setCredentialsProvider(testCredentials); + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + assertTrue(dst.getObjectDatabase().has(A_txt)); + assertEquals(B, dst.exactRef(master).getObjectId()); + fsck(dst, B); + } + + List requests = getRequests(); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); + + AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); - assertEquals(join(authURI, "info/refs"), info.getPath()); - assertEquals(1, info.getParameters().size()); - assertEquals("git-upload-pack", info.getParameter("service")); - assertEquals(200, info.getStatus()); - assertEquals("application/x-git-upload-pack-advertisement", - info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + assertEquals(401, info.getStatus()); - AccessEvent service = requests.get(2); - assertEquals("POST", service.getMethod()); - assertEquals(join(authURI, "git-upload-pack"), service.getPath()); - assertEquals(0, service.getParameters().size()); - assertNotNull("has content-length", - service.getRequestHeader(HDR_CONTENT_LENGTH)); - assertNull("not chunked", - service.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertFetchRequests(requests, 1); + } - assertEquals(200, service.getStatus()); - assertEquals("application/x-git-upload-pack-result", - service.getResponseHeader(HDR_CONTENT_TYPE)); + @Test + public void testInitialClone_PreAuthenticationTooLate() throws Exception { + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, authURI)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password); + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + assertTrue(dst.getObjectDatabase().has(A_txt)); + assertEquals(B, dst.exactRef(master).getObjectId()); + fsck(dst, B); + List requests = getRequests(); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); + assertFetchRequests(requests, 0); + assertThrows(IllegalStateException.class, + () -> ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password)); + assertThrows(IllegalStateException.class, () -> ((TransportHttp) t) + .setPreemptiveBasicAuthentication(null, null)); + } + } + + @Test + public void testInitialClone_WithWrongPreAuthenticationAndCredentialProvider() + throws Exception { + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, authURI)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password + 'x'); + t.setCredentialsProvider(testCredentials); + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + assertTrue(dst.getObjectDatabase().has(A_txt)); + assertEquals(B, dst.exactRef(master).getObjectId()); + fsck(dst, B); + } + + List requests = getRequests(); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); + + AccessEvent info = requests.get(0); + assertEquals("GET", info.getMethod()); + assertEquals(401, info.getStatus()); + + assertFetchRequests(requests, 1); + } + + @Test + public void testInitialClone_WithWrongPreAuthentication() throws Exception { + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, authURI)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password + 'x'); + TransportException e = assertThrows(TransportException.class, + () -> t.fetch(NullProgressMonitor.INSTANCE, + mirror(master))); + String msg = e.getMessage(); + assertTrue("Unexpected exception message: " + msg, + msg.contains("no CredentialsProvider")); + } + List requests = getRequests(); + assertEquals(1, requests.size()); + + AccessEvent info = requests.get(0); + assertEquals("GET", info.getMethod()); + assertEquals(401, info.getStatus()); + } + + @Test + public void testInitialClone_WithUserInfo() throws Exception { + URIish withUserInfo = authURI.setUser(AppServer.username) + .setPass(AppServer.password); + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, withUserInfo)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + assertTrue(dst.getObjectDatabase().has(A_txt)); + assertEquals(B, dst.exactRef(master).getObjectId()); + fsck(dst, B); + } + + List requests = getRequests(); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); + + assertFetchRequests(requests, 0); + } + + @Test + public void testInitialClone_PreAuthOverridesUserInfo() throws Exception { + URIish withUserInfo = authURI.setUser(AppServer.username) + .setPass(AppServer.password + 'x'); + try (Repository dst = createBareRepository(); + Transport t = Transport.open(dst, withUserInfo)) { + assertFalse(dst.getObjectDatabase().has(A_txt)); + ((TransportHttp) t).setPreemptiveBasicAuthentication( + AppServer.username, AppServer.password); + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + assertTrue(dst.getObjectDatabase().has(A_txt)); + assertEquals(B, dst.exactRef(master).getObjectId()); + fsck(dst, B); + } + + List requests = getRequests(); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); + + assertFetchRequests(requests, 0); } @Test @@ -937,19 +1218,20 @@ public boolean get(URIish uri, CredentialItem... items) } List requests = getRequests(); - assertEquals(4, requests.size()); + assertEquals(enableProtocolV2 ? 5 : 4, requests.size()); - AccessEvent redirect = requests.get(0); + int requestNumber = 0; + AccessEvent redirect = requests.get(requestNumber++); assertEquals("GET", redirect.getMethod()); assertEquals(join(cloneFrom, "info/refs"), redirect.getPath()); assertEquals(301, redirect.getStatus()); - AccessEvent info = requests.get(1); + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(authURI, "info/refs"), info.getPath()); assertEquals(401, info.getStatus()); - info = requests.get(2); + info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(authURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -957,9 +1239,24 @@ public boolean get(URIish uri, CredentialItem... items) assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } else { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(authURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } - AccessEvent service = requests.get(3); + AccessEvent service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(authURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -987,7 +1284,7 @@ public void testInitialClone_WithAuthenticationOnPostOnly() } List requests = getRequests(); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); AccessEvent info = requests.get(0); assertEquals("GET", info.getMethod()); @@ -997,25 +1294,30 @@ public void testInitialClone_WithAuthenticationOnPostOnly() assertEquals(200, info.getStatus()); assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); - assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + if (!enableProtocolV2) { + assertEquals("gzip", info.getResponseHeader(HDR_CONTENT_ENCODING)); + } AccessEvent service = requests.get(1); assertEquals("POST", service.getMethod()); assertEquals(join(authOnPostURI, "git-upload-pack"), service.getPath()); assertEquals(401, service.getStatus()); - service = requests.get(2); - assertEquals("POST", service.getMethod()); - assertEquals(join(authOnPostURI, "git-upload-pack"), service.getPath()); - assertEquals(0, service.getParameters().size()); - assertNotNull("has content-length", - service.getRequestHeader(HDR_CONTENT_LENGTH)); - assertNull("not chunked", - service.getRequestHeader(HDR_TRANSFER_ENCODING)); + for (int i = 2; i < requests.size(); i++) { + service = requests.get(i); + assertEquals("POST", service.getMethod()); + assertEquals(join(authOnPostURI, "git-upload-pack"), + service.getPath()); + assertEquals(0, service.getParameters().size()); + assertNotNull("has content-length", + service.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + service.getRequestHeader(HDR_TRANSFER_ENCODING)); - assertEquals(200, service.getStatus()); - assertEquals("application/x-git-upload-pack-result", - service.getResponseHeader(HDR_CONTENT_TYPE)); + assertEquals(200, service.getStatus()); + assertEquals("application/x-git-upload-pack-result", + service.getResponseHeader(HDR_CONTENT_TYPE)); + } } @Test @@ -1052,9 +1354,11 @@ public void testFetch_FewLocalCommits() throws Exception { List requests = getRequests(); requests.removeAll(cloneRequests); - assertEquals(2, requests.size()); - AccessEvent info = requests.get(0); + assertEquals(enableProtocolV2 ? 3 : 2, requests.size()); + + int requestNumber = 0; + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(remoteURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -1063,9 +1367,24 @@ public void testFetch_FewLocalCommits() throws Exception { assertEquals("application/x-git-upload-pack-advertisement", info.getResponseHeader(HDR_CONTENT_TYPE)); + if (enableProtocolV2) { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } + // We should have needed one request to perform the fetch. // - AccessEvent service = requests.get(1); + AccessEvent service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -1116,9 +1435,10 @@ public void testFetch_TooManyLocalCommits() throws Exception { List requests = getRequests(); requests.removeAll(cloneRequests); - assertEquals(3, requests.size()); + assertEquals(enableProtocolV2 ? 4 : 3, requests.size()); - AccessEvent info = requests.get(0); + int requestNumber = 0; + AccessEvent info = requests.get(requestNumber++); assertEquals("GET", info.getMethod()); assertEquals(join(remoteURI, "info/refs"), info.getPath()); assertEquals(1, info.getParameters().size()); @@ -1127,10 +1447,25 @@ public void testFetch_TooManyLocalCommits() throws Exception { assertEquals("application/x-git-upload-pack-advertisement", info .getResponseHeader(HDR_CONTENT_TYPE)); + if (enableProtocolV2) { + AccessEvent lsRefs = requests.get(requestNumber++); + assertEquals("POST", lsRefs.getMethod()); + assertEquals(join(remoteURI, "git-upload-pack"), lsRefs.getPath()); + assertEquals(0, lsRefs.getParameters().size()); + assertNotNull("has content-length", + lsRefs.getRequestHeader(HDR_CONTENT_LENGTH)); + assertNull("not chunked", + lsRefs.getRequestHeader(HDR_TRANSFER_ENCODING)); + assertEquals("version=2", lsRefs.getRequestHeader("Git-Protocol")); + assertEquals(200, lsRefs.getStatus()); + assertEquals("application/x-git-upload-pack-result", + lsRefs.getResponseHeader(HDR_CONTENT_TYPE)); + } + // We should have needed two requests to perform the fetch // due to the high number of local unknown commits. // - AccessEvent service = requests.get(1); + AccessEvent service = requests.get(requestNumber++); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -1143,7 +1478,7 @@ public void testFetch_TooManyLocalCommits() throws Exception { assertEquals("application/x-git-upload-pack-result", service .getResponseHeader(HDR_CONTENT_TYPE)); - service = requests.get(2); + service = requests.get(requestNumber); assertEquals("POST", service.getMethod()); assertEquals(join(remoteURI, "git-upload-pack"), service.getPath()); assertEquals(0, service.getParameters().size()); @@ -1157,6 +1492,64 @@ public void testFetch_TooManyLocalCommits() throws Exception { .getResponseHeader(HDR_CONTENT_TYPE)); } + @Test + public void testFetch_MaxHavesCutoffAfterAckOnly() throws Exception { + // Bootstrap by doing the clone. + // + TestRepository dst = createTestRepository(); + try (Transport t = Transport.open(dst.getRepository(), remoteURI)) { + t.fetch(NullProgressMonitor.INSTANCE, mirror(master)); + } + assertEquals(B, dst.getRepository().exactRef(master).getObjectId()); + + // Force enough into the local client that enumeration will + // need more than MAX_HAVES (256) haves to be sent. The server + // doesn't know any of these, so it will never ACK. The client + // should keep going. + // + // If it does, client and server will find a common commit, and + // the pack file will contain exactly the one commit object Z + // we create on the remote, which we can test for via the progress + // monitor, which should have something like + // "Receiving objects: 100% (1/1)". If the client sends a "done" + // too early, the server will send more objects, and we'll have + // a line like "Receiving objects: 100% (8/8)". + TestRepository.BranchBuilder b = dst.branch(master); + // The client will send 32 + 64 + 128 + 256 + 512 haves. Only the + // last one will be a common commit. If the cutoff kicks in too + // early (after 480), we'll get too many objects in the fetch. + for (int i = 0; i < 992; i++) + b.commit().tick(3600 /* 1 hour */).message("c" + i).create(); + + // Create a new commit on the remote. + // + RevCommit Z; + try (TestRepository tr = new TestRepository<>( + remoteRepository)) { + b = tr.branch(master); + Z = b.commit().message("Z").create(); + } + + // Now incrementally update. + // + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + Writer writer = new OutputStreamWriter(buffer, StandardCharsets.UTF_8); + TextProgressMonitor monitor = new TextProgressMonitor(writer); + try (Transport t = Transport.open(dst.getRepository(), remoteURI)) { + t.fetch(monitor, mirror(master)); + } + assertEquals(Z, dst.getRepository().exactRef(master).getObjectId()); + + String progressMessages = new String(buffer.toByteArray(), + StandardCharsets.UTF_8); + Pattern expected = Pattern + .compile("Receiving objects:\\s+100% \\(1/1\\)\n"); + if (!expected.matcher(progressMessages).find()) { + System.out.println(progressMessages); + fail("Expected only one object to be sent"); + } + } + @Test public void testInitialClone_BrokenServer() throws Exception { try (Repository dst = createBareRepository(); @@ -1211,7 +1604,8 @@ public void testInvalidWant() throws Exception { Collections. emptySet()); fail("Server accepted want " + id.name()); } catch (TransportException err) { - assertEquals("want " + id.name() + " not valid", err.getMessage()); + assertTrue(err.getMessage() + .contains("want " + id.name() + " not valid")); } } @@ -1224,6 +1618,8 @@ public void testFetch_RefsUnreadableOnUpload() throws Exception { new DfsRepositoryDescription(repoName)); final TestRepository repo = new TestRepository<>( badRefsRepo); + badRefsRepo.getConfig().setInt("protocol", null, "version", + enableProtocolV2 ? 2 : 0); ServletContextHandler app = noRefServer.addContext("/git"); GitServlet gs = new GitServlet(); @@ -1253,7 +1649,8 @@ public void testFetch_RefsUnreadableOnUpload() throws Exception { Collections. emptySet()); fail("Successfully served ref with value " + c.getRef(master)); } catch (TransportException err) { - assertEquals("Internal server error", err.getMessage()); + assertTrue("Unexpected exception message " + err.getMessage(), + err.getMessage().contains("Internal server error")); } } finally { noRefServer.tearDown(); @@ -1429,5 +1826,4 @@ private void enableReceivePack() throws IOException { cfg.setBoolean("http", null, "receivepack", true); cfg.save(); } - } diff --git a/org.eclipse.jgit.junit.http/src/org/eclipse/jgit/junit/http/AppServer.java b/org.eclipse.jgit.junit.http/src/org/eclipse/jgit/junit/http/AppServer.java index 4e27a3d35..f9f8c856b 100644 --- a/org.eclipse.jgit.junit.http/src/org/eclipse/jgit/junit/http/AppServer.java +++ b/org.eclipse.jgit.junit.http/src/org/eclipse/jgit/junit/http/AppServer.java @@ -23,8 +23,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.security.AbstractLoginService; @@ -259,7 +259,7 @@ public ServletContextHandler authBasic(ServletContextHandler ctx, static class TestMappedLoginService extends AbstractLoginService { private String role; - protected final ConcurrentMap users = new ConcurrentHashMap<>(); + protected final Map users = new ConcurrentHashMap<>(); TestMappedLoginService(String role) { this.role = role; diff --git a/org.eclipse.jgit.junit.ssh/.settings/edu.umd.cs.findbugs.core.prefs b/org.eclipse.jgit.junit.ssh/.settings/edu.umd.cs.findbugs.core.prefs new file mode 100644 index 000000000..2792ea032 --- /dev/null +++ b/org.eclipse.jgit.junit.ssh/.settings/edu.umd.cs.findbugs.core.prefs @@ -0,0 +1,145 @@ +#SpotBugs User Preferences +#Fri Dec 04 11:26:04 CET 2020 +detectorExplicitSerialization=ExplicitSerialization|true +detectorMultithreadedInstanceAccess=MultithreadedInstanceAccess|true +detectorConfusionBetweenInheritedAndOuterMethod=ConfusionBetweenInheritedAndOuterMethod|true +detectorWrongMapIterator=WrongMapIterator|true +detectorUnnecessaryMath=UnnecessaryMath|true +detectorUselessSubclassMethod=UselessSubclassMethod|false +filter_settings=Medium|BAD_PRACTICE,CORRECTNESS,EXPERIMENTAL,I18N,MALICIOUS_CODE,MT_CORRECTNESS,PERFORMANCE,SECURITY,STYLE|false|15 +detectorURLProblems=URLProblems|true +detectorIteratorIdioms=IteratorIdioms|true +detectorMutableEnum=MutableEnum|true +detectorFindNonShortCircuit=FindNonShortCircuit|true +detectorSynchronizeAndNullCheckField=SynchronizeAndNullCheckField|true +detectorVolatileUsage=VolatileUsage|true +detectorFindNakedNotify=FindNakedNotify|true +detectorFindUninitializedGet=FindUninitializedGet|true +detectorFindUseOfNonSerializableValue=FindUseOfNonSerializableValue|true +detectorFindJSR166LockMonitorenter=FindJSR166LockMonitorenter|true +detectorQuestionableBooleanAssignment=QuestionableBooleanAssignment|true +detectorSwitchFallthrough=SwitchFallthrough|true +detectorFindLocalSelfAssignment2=FindLocalSelfAssignment2|true +detectorConfusedInheritance=ConfusedInheritance|true +detectorSynchronizationOnSharedBuiltinConstant=SynchronizationOnSharedBuiltinConstant|true +detectorMutableStaticFields=MutableStaticFields|true +detectorInvalidJUnitTest=InvalidJUnitTest|true +detectorInfiniteLoop=InfiniteLoop|true +detectorFindRunInvocations=FindRunInvocations|true +detectorBadSyntaxForRegularExpression=BadSyntaxForRegularExpression|true +detectorXMLFactoryBypass=XMLFactoryBypass|true +detectorFindOpenStream=FindOpenStream|true +detectorCheckExpectedWarnings=CheckExpectedWarnings|false +detectorHugeSharedStringConstants=HugeSharedStringConstants|true +detectorLostLoggerDueToWeakReference=LostLoggerDueToWeakReference|true +detectorStringConcatenation=StringConcatenation|true +detectorLoadOfKnownNullValue=LoadOfKnownNullValue|true +detectorFinalizerNullsFields=FinalizerNullsFields|true +detectorFindFieldSelfAssignment=FindFieldSelfAssignment|true +detectorInefficientToArray=InefficientToArray|false +detectorDontCatchIllegalMonitorStateException=DontCatchIllegalMonitorStateException|true +detectorInconsistentAnnotations=InconsistentAnnotations|true +detectorBadlyOverriddenAdapter=BadlyOverriddenAdapter|true +detectorInstantiateStaticClass=InstantiateStaticClass|true +detectorCheckRelaxingNullnessAnnotation=CheckRelaxingNullnessAnnotation|true +detectorMethodReturnCheck=MethodReturnCheck|true +detectorEqualsOperandShouldHaveClassCompatibleWithThis=EqualsOperandShouldHaveClassCompatibleWithThis|true +detectorFindDoubleCheck=FindDoubleCheck|true +detectorFindBadForLoop=FindBadForLoop|true +detectorDefaultEncodingDetector=DefaultEncodingDetector|true +detectorFindInconsistentSync2=FindInconsistentSync2|true +detectorFindSpinLoop=FindSpinLoop|true +detectorFindMaskedFields=FindMaskedFields|true +detectorBooleanReturnNull=BooleanReturnNull|true +detectorFindUnsyncGet=FindUnsyncGet|true +detectorCrossSiteScripting=CrossSiteScripting|true +detectorDroppedException=DroppedException|true +detectorFindDeadLocalStores=FindDeadLocalStores|true +detectorCheckImmutableAnnotation=CheckImmutableAnnotation|true +detectorInfiniteRecursiveLoop=InfiniteRecursiveLoop|true +detectorFindRefComparison=FindRefComparison|true +detectorFindRoughConstants=FindRoughConstants|true +detectorMutableLock=MutableLock|true +detectorFindNullDeref=FindNullDeref|true +detectorFindReturnRef=FindReturnRef|true +detectorSynchronizeOnClassLiteralNotGetClass=SynchronizeOnClassLiteralNotGetClass|true +detectorFindUselessControlFlow=FindUselessControlFlow|true +detectorOverridingEqualsNotSymmetrical=OverridingEqualsNotSymmetrical|true +detectorIDivResultCastToDouble=IDivResultCastToDouble|true +detectorReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass=ReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass|true +detectorFindSelfComparison=FindSelfComparison|true +detectorFindFloatEquality=FindFloatEquality|true +detectorFindComparatorProblems=FindComparatorProblems|true +detectorRepeatedConditionals=RepeatedConditionals|true +filter_settings_neg=NOISE| +detectorInefficientMemberAccess=InefficientMemberAccess|false +detectorFindUncalledPrivateMethods=FindUncalledPrivateMethods|true +detectorNumberConstructor=NumberConstructor|true +detectorDontAssertInstanceofInTests=DontAssertInstanceofInTests|true +detectorFindFinalizeInvocations=FindFinalizeInvocations|true +detectorFindNullDerefsInvolvingNonShortCircuitEvaluation=FindNullDerefsInvolvingNonShortCircuitEvaluation|true +detectorDontIgnoreResultOfPutIfAbsent=DontIgnoreResultOfPutIfAbsent|true +detectorFindUnconditionalWait=FindUnconditionalWait|true +detectorFindTwoLockWait=FindTwoLockWait|true +detectorFindSleepWithLockHeld=FindSleepWithLockHeld|true +detectorFindUnreleasedLock=FindUnreleasedLock|true +detectorInefficientIndexOf=InefficientIndexOf|false +detectorDoInsideDoPrivileged=DoInsideDoPrivileged|true +detectorFindEmptySynchronizedBlock=FindEmptySynchronizedBlock|true +detectorOverridingMethodsMustInvokeSuperDetector=OverridingMethodsMustInvokeSuperDetector|true +detectorWaitInLoop=WaitInLoop|true +detectorIntCast2LongAsInstant=IntCast2LongAsInstant|true +detectorBadUseOfReturnValue=BadUseOfReturnValue|true +detectorFindSqlInjection=FindSqlInjection|true +detectorUnreadFields=UnreadFields|true +detectorSynchronizingOnContentsOfFieldToProtectField=SynchronizingOnContentsOfFieldToProtectField|true +detectorFindUselessObjects=FindUselessObjects|true +detectorBadAppletConstructor=BadAppletConstructor|false +detectorInheritanceUnsafeGetResource=InheritanceUnsafeGetResource|true +detectorSerializableIdiom=SerializableIdiom|true +detectorNaming=Naming|true +detectorNoteUnconditionalParamDerefs=NoteUnconditionalParamDerefs|true +detectorFormatStringChecker=FormatStringChecker|true +detectorSuspiciousThreadInterrupted=SuspiciousThreadInterrupted|true +detectorEmptyZipFileEntry=EmptyZipFileEntry|false +detectorFindCircularDependencies=FindCircularDependencies|false +detectorPreferZeroLengthArrays=PreferZeroLengthArrays|true +detectorAtomicityProblem=AtomicityProblem|true +detectorRuntimeExceptionCapture=RuntimeExceptionCapture|true +detectorInitializationChain=InitializationChain|true +detectorInitializeNonnullFieldsInConstructor=InitializeNonnullFieldsInConstructor|true +detectorOptionalReturnNull=OptionalReturnNull|true +detectorStartInConstructor=StartInConstructor|true +detectorFindUnsatisfiedObligation=FindUnsatisfiedObligation|true +detectorRedundantConditions=RedundantConditions|true +effort=default +detectorRedundantInterfaces=RedundantInterfaces|true +detectorDuplicateBranches=DuplicateBranches|true +detectorCheckTypeQualifiers=CheckTypeQualifiers|true +detectorComparatorIdiom=ComparatorIdiom|true +detectorFindBadCast2=FindBadCast2|true +detectorFindMismatchedWaitOrNotify=FindMismatchedWaitOrNotify|true +excludefilter0=findBugs/FindBugsExcludeFilter.xml|true +detectorBadResultSetAccess=BadResultSetAccess|true +detectorIncompatMask=IncompatMask|true +detectorCovariantArrayAssignment=CovariantArrayAssignment|false +detectorDumbMethodInvocations=DumbMethodInvocations|true +run_at_full_build=false +detectorStaticCalendarDetector=StaticCalendarDetector|true +detectorUncallableMethodOfAnonymousClass=UncallableMethodOfAnonymousClass|true +detectorVarArgsProblems=VarArgsProblems|true +detectorInefficientInitializationInsideLoop=InefficientInitializationInsideLoop|false +detectorCloneIdiom=CloneIdiom|true +detectorFindHEmismatch=FindHEmismatch|true +detectorAppendingToAnObjectOutputStream=AppendingToAnObjectOutputStream|true +detectorFindSelfComparison2=FindSelfComparison2|true +detectorLazyInit=LazyInit|true +detectorFindUnrelatedTypesInGenericContainer=FindUnrelatedTypesInGenericContainer|true +detectorDontUseEnum=DontUseEnum|true +detectorFindPuzzlers=FindPuzzlers|true +detectorCallToUnsupportedMethod=CallToUnsupportedMethod|false +detectorSuperfluousInstanceOf=SuperfluousInstanceOf|true +detectorReadReturnShouldBeChecked=ReadReturnShouldBeChecked|true +detector_threshold=2 +detectorPublicSemaphores=PublicSemaphores|false +detectorDumbMethods=DumbMethods|true diff --git a/org.eclipse.jgit.junit.ssh/META-INF/MANIFEST.MF b/org.eclipse.jgit.junit.ssh/META-INF/MANIFEST.MF index 8858c09cb..a347715d7 100644 --- a/org.eclipse.jgit.junit.ssh/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.junit.ssh/META-INF/MANIFEST.MF @@ -8,28 +8,31 @@ Bundle-Localization: plugin Bundle-Vendor: %Bundle-Vendor Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.apache.sshd.common;version="[2.4.0,2.5.0)", - org.apache.sshd.common.config.keys;version="[2.4.0,2.5.0)", - org.apache.sshd.common.file.virtualfs;version="[2.4.0,2.5.0)", - org.apache.sshd.common.helpers;version="[2.4.0,2.5.0)", - org.apache.sshd.common.io;version="[2.4.0,2.5.0)", - org.apache.sshd.common.kex;version="[2.4.0,2.5.0)", - org.apache.sshd.common.keyprovider;version="[2.4.0,2.5.0)", - org.apache.sshd.common.session;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.buffer;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.logging;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.security;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.threads;version="[2.4.0,2.5.0)", - org.apache.sshd.server;version="[2.4.0,2.5.0)", - org.apache.sshd.server.auth;version="[2.4.0,2.5.0)", - org.apache.sshd.server.auth.gss;version="[2.4.0,2.5.0)", - org.apache.sshd.server.auth.keyboard;version="[2.4.0,2.5.0)", - org.apache.sshd.server.auth.password;version="[2.4.0,2.5.0)", - org.apache.sshd.server.command;version="[2.4.0,2.5.0)", - org.apache.sshd.server.session;version="[2.4.0,2.5.0)", - org.apache.sshd.server.shell;version="[2.4.0,2.5.0)", - org.apache.sshd.server.subsystem;version="[2.4.0,2.5.0)", - org.apache.sshd.server.subsystem.sftp;version="[2.4.0,2.5.0)", +Import-Package: org.apache.sshd.common;version="[2.6.0,2.7.0)", + org.apache.sshd.common.config.keys;version="[2.6.0,2.7.0)", + org.apache.sshd.common.file.virtualfs;version="[2.6.0,2.7.0)", + org.apache.sshd.common.helpers;version="[2.6.0,2.7.0)", + org.apache.sshd.common.io;version="[2.6.0,2.7.0)", + org.apache.sshd.common.kex;version="[2.6.0,2.7.0)", + org.apache.sshd.common.keyprovider;version="[2.6.0,2.7.0)", + org.apache.sshd.common.session;version="[2.6.0,2.7.0)", + org.apache.sshd.common.signature;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.buffer;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.logging;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.security;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.threads;version="[2.6.0,2.7.0)", + org.apache.sshd.core;version="[2.6.0,2.7.0)", + org.apache.sshd.server;version="[2.6.0,2.7.0)", + org.apache.sshd.server.auth;version="[2.6.0,2.7.0)", + org.apache.sshd.server.auth.gss;version="[2.6.0,2.7.0)", + org.apache.sshd.server.auth.keyboard;version="[2.6.0,2.7.0)", + org.apache.sshd.server.auth.password;version="[2.6.0,2.7.0)", + org.apache.sshd.server.command;version="[2.6.0,2.7.0)", + org.apache.sshd.server.session;version="[2.6.0,2.7.0)", + org.apache.sshd.server.shell;version="[2.6.0,2.7.0)", + org.apache.sshd.server.subsystem;version="[2.6.0,2.7.0)", + org.apache.sshd.sftp;version="[2.6.0,2.7.0)", + org.apache.sshd.sftp.server;version="[2.6.0,2.7.0)", org.eclipse.jgit.annotations;version="[6.0.0,6.1.0)", org.eclipse.jgit.api;version="[6.0.0,6.1.0)", org.eclipse.jgit.api.errors;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.junit.ssh/findBugs/FindBugsExcludeFilter.xml b/org.eclipse.jgit.junit.ssh/findBugs/FindBugsExcludeFilter.xml new file mode 100644 index 000000000..999cb71f7 --- /dev/null +++ b/org.eclipse.jgit.junit.ssh/findBugs/FindBugsExcludeFilter.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshBasicTestBase.java b/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshBasicTestBase.java new file mode 100644 index 000000000..f9ca0b892 --- /dev/null +++ b/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshBasicTestBase.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.junit.ssh; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; + +import org.eclipse.jgit.api.Git; +import org.junit.Test; + +/** + * Some minimal cloning and fetching tests. Concrete subclasses can implement + * the abstract operations from {@link SshTestHarness} to run with different SSH + * implementations. + */ +public abstract class SshBasicTestBase extends SshTestHarness { + + protected File defaultCloneDir; + + @Override + public void setUp() throws Exception { + super.setUp(); + defaultCloneDir = new File(getTemporaryDirectory(), "cloned"); + } + + @Test + public void testSshCloneWithConfig() throws Exception { + cloneWith("ssh://localhost/doesntmatter", defaultCloneDir, null, // + "Host localhost", // + "HostName localhost", // + "Port " + testPort, // + "User " + TEST_USER, // + "IdentityFile " + privateKey1.getAbsolutePath()); + } + + @Test + public void testSshFetchWithConfig() throws Exception { + File localClone = cloneWith("ssh://localhost/doesntmatter", + defaultCloneDir, null, // + "Host localhost", // + "HostName localhost", // + "Port " + testPort, // + "User " + TEST_USER, // + "IdentityFile " + privateKey1.getAbsolutePath()); + // Do a commit in the upstream repo + try (Git git = new Git(db)) { + writeTrashFile("SomeOtherFile.txt", "Other commit"); + git.add().addFilepattern("SomeOtherFile.txt").call(); + git.commit().setMessage("New commit").call(); + } + // Pull in the clone + try (Git git = Git.open(localClone)) { + File f = new File(git.getRepository().getWorkTree(), + "SomeOtherFile.txt"); + assertFalse(f.exists()); + git.pull().setRemote("origin").call(); + assertTrue(f.exists()); + assertEquals("Other commit", read(f)); + } + } +} diff --git a/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestBase.java b/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestBase.java index 378474119..6fa82f1d6 100644 --- a/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestBase.java +++ b/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestBase.java @@ -40,7 +40,7 @@ * abstract operations from {@link SshTestHarness}. This gives a way to test * different ssh clients against a unified test suite. */ -public abstract class SshTestBase extends SshTestHarness { +public abstract class SshTestBase extends SshBasicTestBase { @DataPoints public static String[] KEY_RESOURCES = { // @@ -65,14 +65,6 @@ public abstract class SshTestBase extends SshTestHarness { "id_ed25519_testpass", // "id_ed25519_expensive_testpass" }; - protected File defaultCloneDir; - - @Override - public void setUp() throws Exception { - super.setUp(); - defaultCloneDir = new File(getTemporaryDirectory(), "cloned"); - } - @Test public void testSshWithoutConfig() throws Exception { assertThrows(TransportException.class, @@ -132,16 +124,6 @@ public void testSshWithDefaultIdentity() throws Exception { + "/doesntmatter", defaultCloneDir, null); } - @Test - public void testSshWithConfig() throws Exception { - cloneWith("ssh://localhost/doesntmatter", defaultCloneDir, null, // - "Host localhost", // - "HostName localhost", // - "Port " + testPort, // - "User " + TEST_USER, // - "IdentityFile " + privateKey1.getAbsolutePath()); - } - @Test public void testSshWithConfigEncryptedUnusedKey() throws Exception { // Copy the encrypted test key from the bundle. diff --git a/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestGitServer.java b/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestGitServer.java index ab8e0c1ca..4fe98f868 100644 --- a/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestGitServer.java +++ b/org.eclipse.jgit.junit.ssh/src/org/eclipse/jgit/junit/ssh/SshTestGitServer.java @@ -9,6 +9,9 @@ */ package org.eclipse.jgit.junit.ssh; +import static org.apache.sshd.core.CoreModuleProperties.SERVER_EXTRA_IDENTIFICATION_LINES; +import static org.apache.sshd.core.CoreModuleProperties.SERVER_EXTRA_IDENT_LINES_SEPARATOR; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -21,26 +24,28 @@ import java.security.PublicKey; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; +import org.apache.sshd.common.NamedFactory; import org.apache.sshd.common.NamedResource; import org.apache.sshd.common.PropertyResolver; -import org.apache.sshd.common.PropertyResolverUtils; import org.apache.sshd.common.SshConstants; import org.apache.sshd.common.config.keys.AuthorizedKeyEntry; import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.common.config.keys.PublicKeyEntryResolver; import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory; -import org.apache.sshd.common.session.Session; +import org.apache.sshd.common.signature.BuiltinSignatures; +import org.apache.sshd.common.signature.Signature; import org.apache.sshd.common.util.buffer.Buffer; import org.apache.sshd.common.util.security.SecurityUtils; import org.apache.sshd.common.util.threads.CloseableExecutorService; import org.apache.sshd.common.util.threads.ThreadUtils; import org.apache.sshd.server.ServerAuthenticationManager; -import org.apache.sshd.server.ServerFactoryManager; +import org.apache.sshd.server.ServerBuilder; import org.apache.sshd.server.SshServer; import org.apache.sshd.server.auth.UserAuth; import org.apache.sshd.server.auth.UserAuthFactory; @@ -52,8 +57,9 @@ import org.apache.sshd.server.session.ServerSession; import org.apache.sshd.server.shell.UnknownCommand; import org.apache.sshd.server.subsystem.SubsystemFactory; -import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory; +import org.apache.sshd.sftp.server.SftpSubsystemFactory; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.ReceivePack; import org.eclipse.jgit.transport.RemoteConfig; @@ -161,7 +167,9 @@ public SshTestGitServer(@NonNull String testUser, this.testUser = testUser; setTestUserPublicKey(testKey); this.repository = repository; - server = SshServer.setUpDefaultServer(); + ServerBuilder builder = ServerBuilder.builder() + .signatureFactories(getSignatureFactories()); + server = builder.build(); hostKeys.add(hostKey); server.setKeyPairProvider((session) -> hostKeys); @@ -186,6 +194,37 @@ public SshTestGitServer(@NonNull String testUser, }); } + /** + * Apache MINA sshd 2.6.0 has removed DSA, DSA_CERT and RSA_CERT. We have to + * set it up explicitly to still allow users to connect with DSA keys. + * + * @return a list of supported signature factories + */ + @SuppressWarnings("deprecation") + private static List> getSignatureFactories() { + // @formatter:off + return Arrays.asList( + BuiltinSignatures.nistp256_cert, + BuiltinSignatures.nistp384_cert, + BuiltinSignatures.nistp521_cert, + BuiltinSignatures.ed25519_cert, + BuiltinSignatures.rsaSHA512_cert, + BuiltinSignatures.rsaSHA256_cert, + BuiltinSignatures.rsa_cert, + BuiltinSignatures.nistp256, + BuiltinSignatures.nistp384, + BuiltinSignatures.nistp521, + BuiltinSignatures.ed25519, + BuiltinSignatures.sk_ecdsa_sha2_nistp256, + BuiltinSignatures.sk_ssh_ed25519, + BuiltinSignatures.rsaSHA512, + BuiltinSignatures.rsaSHA256, + BuiltinSignatures.rsa, + BuiltinSignatures.dsa_cert, + BuiltinSignatures.dsa); + // @formatter:on + } + private static PublicKey readPublicKey(Path key) throws IOException, GeneralSecurityException { return AuthorizedKeyEntry.readAuthorizedKeys(key).get(0) @@ -202,7 +241,7 @@ private static KeyPair readKeyPair(byte[] keyMaterial) private static class FakeUserAuthGSS extends UserAuthGSS { @Override - protected Boolean doAuth(Buffer buffer, boolean initial) + protected @Nullable Boolean doAuth(Buffer buffer, boolean initial) throws Exception { // We always reply that we did do this, but then we fail at the // first token message. That way we can test that the client-side @@ -277,14 +316,8 @@ public boolean validateInitialUser(ServerSession session, @NonNull protected List configureSubsystems() { // SFTP. - server.setFileSystemFactory(new VirtualFileSystemFactory() { - - @Override - protected Path computeRootDir(Session session) throws IOException { - return SshTestGitServer.this.repository.getDirectory() - .getParentFile().getAbsoluteFile().toPath(); - } - }); + server.setFileSystemFactory(new VirtualFileSystemFactory(repository + .getDirectory().getParentFile().getAbsoluteFile().toPath())); return Collections .singletonList((new SftpSubsystemFactory.Builder()).build()); } @@ -433,9 +466,8 @@ public void setTestUserPublicKey(@NonNull PublicKey key) { */ public void setPreamble(String... lines) { if (lines != null && lines.length > 0) { - PropertyResolverUtils.updateProperty(this.server, - ServerFactoryManager.SERVER_EXTRA_IDENTIFICATION_LINES, - String.join("|", lines)); + SERVER_EXTRA_IDENTIFICATION_LINES.set(server, String.join( + String.valueOf(SERVER_EXTRA_IDENT_LINES_SEPARATOR), lines)); } } diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java index b982787e7..4a4dc92c4 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/SeparateClassloaderTestRunner.java @@ -9,10 +9,10 @@ */ package org.eclipse.jgit.junit; -import static java.lang.ClassLoader.getSystemClassLoader; - +import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Paths; import org.junit.runners.BlockJUnit4ClassRunner; import org.junit.runners.model.InitializationError; @@ -40,7 +40,13 @@ public SeparateClassloaderTestRunner(Class klass) private static Class loadNewClass(Class klass) throws InitializationError { try { - URL[] urls = ((URLClassLoader) getSystemClassLoader()).getURLs(); + String pathSeparator = System.getProperty("path.separator"); + String[] classPathEntries = System.getProperty("java.class.path") + .split(pathSeparator); + URL[] urls = new URL[classPathEntries.length]; + for (int i = 0; i < classPathEntries.length; i++) { + urls[i] = Paths.get(classPathEntries[i]).toUri().toURL(); + } ClassLoader testClassLoader = new URLClassLoader(urls) { @Override @@ -54,7 +60,7 @@ public Class loadClass(String name) } }; return Class.forName(klass.getName(), true, testClassLoader); - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | MalformedURLException e) { throw new InitializationError(e); } } diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java index a5b3b1f3a..e3eb2c536 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/TestRepository.java @@ -43,7 +43,7 @@ import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.internal.storage.file.ObjectDirectory; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; @@ -773,7 +773,7 @@ protected void writeFile(String name, byte[] bin) rw.writeInfoRefs(); final StringBuilder w = new StringBuilder(); - for (PackFile p : fr.getObjectDatabase().getPacks()) { + for (Pack p : fr.getObjectDatabase().getPacks()) { w.append("P "); w.append(p.getPackFile().getName()); w.append('\n'); @@ -954,7 +954,7 @@ public void close() { } private static void prunePacked(ObjectDirectory odb) throws IOException { - for (PackFile p : odb.getPacks()) { + for (Pack p : odb.getPacks()) { for (MutableEntry e : p) FileUtils.delete(odb.fileFor(e.toObjectId())); } diff --git a/org.eclipse.jgit.lfs.server.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.server.test/META-INF/MANIFEST.MF index 349c3b371..c44152937 100644 --- a/org.eclipse.jgit.lfs.server.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs.server.test/META-INF/MANIFEST.MF @@ -9,11 +9,11 @@ Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: javax.servlet;version="[3.1.0,4.0.0)", javax.servlet.http;version="[3.1.0,4.0.0)", - org.apache.http;version="[4.3.0,5.0.0)", - org.apache.http.client;version="[4.3.0,5.0.0)", - org.apache.http.client.methods;version="[4.3.0,5.0.0)", - org.apache.http.entity;version="[4.3.0,5.0.0)", - org.apache.http.impl.client;version="[4.3.0,5.0.0)", + org.apache.http;version="[4.4.0,5.0.0)", + org.apache.http.client;version="[4.4.0,5.0.0)", + org.apache.http.client.methods;version="[4.4.0,5.0.0)", + org.apache.http.entity;version="[4.4.0,5.0.0)", + org.apache.http.impl.client;version="[4.4.0,5.0.0)", org.eclipse.jetty.continuation;version="[9.4.5,10.0.0)", org.eclipse.jetty.http;version="[9.4.5,10.0.0)", org.eclipse.jetty.io;version="[9.4.5,10.0.0)", diff --git a/org.eclipse.jgit.lfs.server/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.server/META-INF/MANIFEST.MF index df115dba0..76050e117 100644 --- a/org.eclipse.jgit.lfs.server/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs.server/META-INF/MANIFEST.MF @@ -24,7 +24,6 @@ Import-Package: com.google.gson;version="[2.8.0,3.0.0)", javax.servlet.annotation;version="[3.1.0,4.0.0)", javax.servlet.http;version="[3.1.0,4.0.0)", org.apache.http;version="[4.3.0,5.0.0)", - org.apache.http.client;version="[4.3.0,5.0.0)", org.eclipse.jgit.annotations;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.file;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.lfs.server/findBugs/FindBugsExcludeFilter.xml b/org.eclipse.jgit.lfs.server/findBugs/FindBugsExcludeFilter.xml new file mode 100644 index 000000000..c5f3f8013 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/findBugs/FindBugsExcludeFilter.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..b853c6a7e 100644 --- a/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java index 7ee898fab..da78b28d9 100644 --- a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/lib/LFSPointerTest.java @@ -12,7 +12,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -23,17 +29,275 @@ * Test LfsPointer file abstraction */ public class LFSPointerTest { + + private static final String TEST_SHA256 = "27e15b72937fc8f558da24ac3d50ec20302a4cf21e33b87ae8e4ce90e89c4b10"; + @Test public void testEncoding() throws IOException { - final String s = "27e15b72937fc8f558da24ac3d50ec20302a4cf21e33b87ae8e4ce90e89c4b10"; - AnyLongObjectId id = LongObjectId.fromString(s); + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); LfsPointer ptr = new LfsPointer(id, 4); try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { ptr.encode(baos); assertEquals( "version https://git-lfs.github.com/spec/v1\noid sha256:" - + s + "\nsize 4\n", + + TEST_SHA256 + "\nsize 4\n", baos.toString(UTF_8.name())); } } + + @Test + public void testReadValidLfsPointer() throws Exception { + String ptr = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:" + TEST_SHA256 + '\n' + + "size 4\n"; + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertEquals(lfs, LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadValidLfsPointerUnordered() throws Exception { + // This is actually not allowed per the spec, but JGit accepts it + // anyway. + String ptr = "version https://git-lfs.github.com/spec/v1\n" + + "size 4\n" + + "oid sha256:" + TEST_SHA256 + '\n'; + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertEquals(lfs, LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadValidLfsPointerVersionNotFirst() throws Exception { + // This is actually not allowed per the spec, but JGit accepts it + // anyway. + String ptr = "oid sha256:" + TEST_SHA256 + '\n' + + "size 4\n" + + "version https://git-lfs.github.com/spec/v1\n"; + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertEquals(lfs, LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInvalidLfsPointer() throws Exception { + String cSource = "size_t someFunction(void *ptr); // Fake C source\n"; + try (ByteArrayInputStream in = new ByteArrayInputStream( + cSource.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInvalidLfsPointer2() throws Exception { + String cSource = "size_t\nsomeFunction(void *ptr);\n// Fake C source\n"; + try (ByteArrayInputStream in = new ByteArrayInputStream( + cSource.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInValidLfsPointerVersionWrong() throws Exception { + String ptr = "version https://git-lfs.example.org/spec/v1\n" + + "oid sha256:" + TEST_SHA256 + '\n' + + "size 4\n"; + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInValidLfsPointerVersionTwice() throws Exception { + String ptr = "version https://git-lfs.github.com/spec/v1\n" + + "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:" + TEST_SHA256 + '\n' + + "size 4\n"; + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInValidLfsPointerVersionTwice2() throws Exception { + String ptr = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:" + TEST_SHA256 + '\n' + + "version https://git-lfs.github.com/spec/v1\n" + + "size 4\n"; + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInValidLfsPointerOidTwice() throws Exception { + String ptr = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:" + TEST_SHA256 + '\n' + + "oid sha256:" + TEST_SHA256 + '\n' + + "size 4\n"; + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testReadInValidLfsPointerSizeTwice() throws Exception { + String ptr = "version https://git-lfs.github.com/spec/v1\n" + + "size 4\n" + + "size 4\n" + + "oid sha256:" + TEST_SHA256 + '\n'; + try (ByteArrayInputStream in = new ByteArrayInputStream( + ptr.getBytes(UTF_8))) { + assertNull("Is not a LFS pointer", LfsPointer.parseLfsPointer(in)); + } + } + + @Test + public void testRoundtrip() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer ptr = new LfsPointer(id, 4); + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ptr.encode(baos); + try (ByteArrayInputStream in = new ByteArrayInputStream( + baos.toByteArray())) { + assertEquals(ptr, LfsPointer.parseLfsPointer(in)); + } + } + } + + @Test + public void testEquals() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs2 = new LfsPointer(id2, 4); + assertTrue(lfs.equals(lfs2)); + assertTrue(lfs2.equals(lfs)); + } + + @Test + public void testEqualsNull() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + assertFalse(lfs.equals(null)); + } + + @Test + public void testEqualsSame() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + assertTrue(lfs.equals(lfs)); + } + + @Test + public void testEqualsOther() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + assertFalse(lfs.equals(new Object())); + } + + @Test + public void testNotEqualsOid() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId + .fromString(TEST_SHA256.replace('7', '5')); + LfsPointer lfs2 = new LfsPointer(id2, 4); + assertFalse(lfs.equals(lfs2)); + assertFalse(lfs2.equals(lfs)); + } + + @Test + public void testNotEqualsSize() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs2 = new LfsPointer(id2, 5); + assertFalse(lfs.equals(lfs2)); + assertFalse(lfs2.equals(lfs)); + } + + @Test + public void testCompareToEquals() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs2 = new LfsPointer(id2, 4); + assertEquals(0, lfs.compareTo(lfs2)); + assertEquals(0, lfs2.compareTo(lfs)); + } + + @Test + @SuppressWarnings("SelfComparison") + public void testCompareToSame() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + assertEquals(0, lfs.compareTo(lfs)); + } + + @Test + public void testCompareToNotEqualsOid() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId + .fromString(TEST_SHA256.replace('7', '5')); + LfsPointer lfs2 = new LfsPointer(id2, 4); + assertNotEquals(0, lfs.compareTo(lfs2)); + assertNotEquals(0, lfs2.compareTo(lfs)); + } + + @Test + public void testCompareToNotEqualsSize() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs2 = new LfsPointer(id2, 5); + assertNotEquals(0, lfs.compareTo(lfs2)); + assertNotEquals(0, lfs2.compareTo(lfs)); + } + + @Test + public void testCompareToNull() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + assertThrows(NullPointerException.class, () -> lfs.compareTo(null)); + } + + @Test + public void testHashcodeEquals() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs2 = new LfsPointer(id2, 4); + assertEquals(lfs.hashCode(), lfs2.hashCode()); + } + + @Test + public void testHashcodeSame() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + assertEquals(lfs.hashCode(), lfs.hashCode()); + } + + @Test + public void testHashcodeNotEquals() throws Exception { + AnyLongObjectId id = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs = new LfsPointer(id, 4); + AnyLongObjectId id2 = LongObjectId.fromString(TEST_SHA256); + LfsPointer lfs2 = new LfsPointer(id2, 5); + assertNotEquals(lfs.hashCode(), lfs2.hashCode()); + } } diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index 027608831..3eb5a7671 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -13,8 +13,6 @@ Export-Package: org.eclipse.jgit.lfs;version="6.0.0", Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: com.google.gson;version="[2.8.2,3.0.0)", com.google.gson.stream;version="[2.8.2,3.0.0)", - org.apache.http.impl.client;version="[4.2.6,5.0.0)", - org.apache.http.impl.conn;version="[4.2.6,5.0.0)", org.eclipse.jgit.annotations;version="[6.0.0,6.1.0)";resolution:=optional, org.eclipse.jgit.api.errors;version="[6.0.0,6.1.0)", org.eclipse.jgit.attributes;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java index 4e2d8a998..aef441638 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016, Christian Halstrick and others + * Copyright (C) 2016, 2021 Christian Halstrick and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -19,6 +19,7 @@ import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.util.Locale; +import java.util.Objects; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.lfs.lib.AnyLongObjectId; @@ -56,9 +57,9 @@ public class LfsPointer implements Comparable { public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION .toLowerCase(Locale.ROOT).replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$ - private AnyLongObjectId oid; + private final AnyLongObjectId oid; - private long size; + private final long size; /** *

    Constructor for LfsPointer.

    @@ -129,19 +130,49 @@ public static LfsPointer parseLfsPointer(InputStream in) LongObjectId id = null; long sz = -1; + // This parsing is a bit too general if we go by the spec at + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + // Comment lines are not mentioned in the spec, and the "version" line + // MUST be the first. try (BufferedReader br = new BufferedReader( new InputStreamReader(in, UTF_8))) { for (String s = br.readLine(); s != null; s = br.readLine()) { if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ continue; - } else if (s.startsWith("version") && s.length() > 8 //$NON-NLS-1$ - && (s.substring(8).trim().equals(VERSION) || - s.substring(8).trim().equals(VERSION_LEGACY))) { - versionLine = true; - } else if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ - id = LongObjectId.fromString(s.substring(11).trim()); - } else if (s.startsWith("size") && s.length() > 5) { //$NON-NLS-1$ - sz = Long.parseLong(s.substring(5).trim()); + } else if (s.startsWith("version")) { //$NON-NLS-1$ + if (versionLine || s.length() < 8 || s.charAt(7) != ' ') { + return null; // Not a LFS pointer + } + String rest = s.substring(8).trim(); + versionLine = VERSION.equals(rest) + || VERSION_LEGACY.equals(rest); + if (!versionLine) { + return null; // Not a LFS pointer + } + } else { + try { + if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ + if (id != null) { + return null; // Not a LFS pointer + } + id = LongObjectId + .fromString(s.substring(11).trim()); + } else if (s.startsWith("size")) { //$NON-NLS-1$ + if (sz > 0 || s.length() < 5 + || s.charAt(4) != ' ') { + return null; // Not a LFS pointer + } + sz = Long.parseLong(s.substring(5).trim()); + } + } catch (RuntimeException e) { + // We could not parse the line. If we have a version + // already, this is a corrupt LFS pointer. Otherwise it + // is just not an LFS pointer. + if (versionLine) { + throw e; + } + return null; + } } } if (versionLine && id != null && sz > -1) { @@ -170,5 +201,22 @@ public int compareTo(LfsPointer o) { return Long.compare(getSize(), o.getSize()); } -} + @Override + public int hashCode() { + return Objects.hash(getOid()) * 31 + Long.hashCode(getSize()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + LfsPointer other = (LfsPointer) obj; + return Objects.equals(getOid(), other.getOid()) + && getSize() == other.getSize(); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java index 7a0ed456c..e221913be 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java @@ -18,7 +18,9 @@ import java.net.ProxySelector; import java.net.URISyntaxException; import java.net.URL; -import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.LinkedList; import java.util.Map; import java.util.TreeMap; @@ -258,8 +260,8 @@ public static Protocol.Request toRequest(String operation, private static final class AuthCache { private static final long AUTH_CACHE_EAGER_TIMEOUT = 500; - private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$ + private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$ /** * Creates a cache entry for an authentication response. @@ -278,8 +280,10 @@ public AuthCache(Protocol.ExpiringAction action) { - AUTH_CACHE_EAGER_TIMEOUT; } else if (action.expiresAt != null && !action.expiresAt.isEmpty()) { - this.validUntil = ISO_FORMAT.parse(action.expiresAt) - .getTime() - AUTH_CACHE_EAGER_TIMEOUT; + this.validUntil = LocalDateTime + .parse(action.expiresAt, ISO_FORMAT) + .atZone(ZoneOffset.UTC).toInstant().toEpochMilli() + - AUTH_CACHE_EAGER_TIMEOUT; } else { this.validUntil = System.currentTimeMillis(); } diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.feature/feature.xml index 312c62f57..526e8b331 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.feature/feature.xml @@ -36,25 +36,4 @@ version="0.0.0" unpack="false"/> - - - - - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.gpg.bc.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.gpg.bc.feature/feature.xml index 0c7d0bb8d..6883afc34 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.gpg.bc.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.gpg.bc.feature/feature.xml @@ -34,25 +34,4 @@ fragment="true" unpack="false"/> - - - - - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.http.apache.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.http.apache.feature/feature.xml index 8e7bef4b7..22f995e8d 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.http.apache.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.http.apache.feature/feature.xml @@ -33,25 +33,4 @@ version="0.0.0" unpack="false"/> - - - - - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.lfs.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.lfs.feature/feature.xml index 19360b4df..4b5032e45 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.lfs.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.lfs.feature/feature.xml @@ -40,11 +40,4 @@ version="0.0.0" unpack="false"/> - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.pgm.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.pgm.feature/feature.xml index 10529823e..d97ce7552 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.pgm.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.pgm.feature/feature.xml @@ -54,25 +54,4 @@ version="0.0.0" unpack="false"/> - - - - - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.repository/category.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.repository/category.xml index abda69563..a56cf0a1f 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.repository/category.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.repository/category.xml @@ -39,6 +39,138 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Java implementation of Git @@ -49,4 +181,5 @@ Java implementation of Git - additional bundles + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.apache.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.apache.feature/feature.xml index 4efafc2f3..46ea77995 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.apache.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.apache.feature/feature.xml @@ -33,25 +33,4 @@ version="0.0.0" unpack="false"/> - - - - - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.jsch.feature/feature.xml b/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.jsch.feature/feature.xml index 50ef7c30a..06a6ed91e 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.jsch.feature/feature.xml +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.ssh.jsch.feature/feature.xml @@ -34,18 +34,4 @@ fragment="true" unpack="false"/> - - - - diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target index 931e196cf..68378a2b0 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd index b87917b8a..fb1ac6b25 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.tpd @@ -1,7 +1,7 @@ target "jgit-4.10" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2018-12/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target index 809099960..18d525d8e 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd index 844a1d791..0d5628063 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.tpd @@ -1,7 +1,7 @@ target "jgit-4.11" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2019-03/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target index 69eb63972..d72f08d29 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd index afbf79d53..5a024152d 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.tpd @@ -1,7 +1,7 @@ target "jgit-4.12" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2019-06/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target index fdbed2d2a..d0e559271 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd index d83e33846..84e5c25ef 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.tpd @@ -1,7 +1,7 @@ target "jgit-4.13" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2019-09/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target index c2ef9d24a..42278f6ef 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd index f357ccd24..6d793a607 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.tpd @@ -1,7 +1,7 @@ target "jgit-4.14" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2019-12/201912181000/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target index 4034d2a36..0d5166e16 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd index 881fe37e7..4ce832bf9 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.tpd @@ -1,7 +1,7 @@ target "jgit-4.15" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2020-03/202003181000/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target index 34d872cdd..b4d53069b 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd index 9a0759711..1b56447ce 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.tpd @@ -1,7 +1,7 @@ target "jgit-4.16" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2020-06/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target index 3384c2356..47fc74be6 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd index ce79cf45e..367020ce0 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.tpd @@ -1,7 +1,7 @@ target "jgit-4.17" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2020-09/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18-staging.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18-staging.tpd deleted file mode 100644 index 0669490bb..000000000 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18-staging.tpd +++ /dev/null @@ -1,8 +0,0 @@ -target "jgit-4.18-staging" with source configurePhase - -include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" - -location "https://download.eclipse.org/staging/2020-12/" { - org.eclipse.osgi lazy -} diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target new file mode 100644 index 000000000..b393e6075 --- /dev/null +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target @@ -0,0 +1,96 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd new file mode 100644 index 000000000..507ddd1dc --- /dev/null +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.tpd @@ -0,0 +1,8 @@ +target "jgit-4.18" with source configurePhase + +include "projects/jetty-9.4.x.tpd" +include "orbit/S20210216215844.tpd" + +location "https://download.eclipse.org/releases/2020-12/" { + org.eclipse.osgi lazy +} diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18-staging.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target similarity index 71% rename from org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18-staging.target rename to org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target index 5eeab9534..f37692616 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18-staging.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,11 +86,11 @@ - + - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd new file mode 100644 index 000000000..3b1b19c84 --- /dev/null +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19-staging.tpd @@ -0,0 +1,8 @@ +target "jgit-4.19-staging" with source configurePhase + +include "projects/jetty-9.4.x.tpd" +include "orbit/S20210216215844.tpd" + +location "https://download.eclipse.org/staging/2021-03/" { + org.eclipse.osgi lazy +} diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target index a66fcc082..26715ee18 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd index aa58b6857..23bf87c07 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.tpd @@ -1,7 +1,7 @@ target "jgit-4.6" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/neon/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target index 4b5410a3d..64fe05495 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd index e2264e003..c33e4a39b 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.tpd @@ -1,7 +1,7 @@ target "jgit-4.7" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/oxygen/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target index d77642745..f7a3a3b26 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd index c92ce5396..c40bacdb8 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.tpd @@ -1,7 +1,7 @@ target "jgit-4.8" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/photon/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target index 56002b78a..4afbe9973 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target @@ -1,30 +1,32 @@ - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - + + @@ -47,16 +49,16 @@ - - - - + + + + - - - - + + + + @@ -84,7 +86,7 @@ - + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd index 06ccecbd5..5aa63be64 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.tpd @@ -1,7 +1,7 @@ target "jgit-4.9" with source configurePhase include "projects/jetty-9.4.x.tpd" -include "orbit/S20201118210000.tpd" +include "orbit/S20210216215844.tpd" location "https://download.eclipse.org/releases/2018-09/" { org.eclipse.osgi lazy diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20201118210000.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20201130205003-2020-12.tpd similarity index 97% rename from org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20201118210000.tpd rename to org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20201130205003-2020-12.tpd index a00a5e7c8..08a0846de 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20201118210000.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/R20201130205003-2020-12.tpd @@ -1,7 +1,7 @@ -target "S20201118210000" with source configurePhase +target "R20201130205003-2020-12" with source configurePhase // see https://download.eclipse.org/tools/orbit/downloads/ -location "https://download.eclipse.org/tools/orbit/downloads/drops/S20201118210000/repository" { +location "https://download.eclipse.org/tools/orbit/downloads/drops/R20201130205003/repository" { com.google.gson [2.8.2.v20180104-1110,2.8.2.v20180104-1110] com.google.gson.source [2.8.2.v20180104-1110,2.8.2.v20180104-1110] com.jcraft.jsch [0.1.55.v20190404-1902,0.1.55.v20190404-1902] diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20210216215844.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20210216215844.tpd new file mode 100644 index 000000000..29e5bc800 --- /dev/null +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/orbit/S20210216215844.tpd @@ -0,0 +1,66 @@ +target "S20210216215844" with source configurePhase +// see https://download.eclipse.org/tools/orbit/downloads/ + +location "https://download.eclipse.org/tools/orbit/downloads/drops/S20210216215844/repository" { + com.google.gson [2.8.6.v20201231-1626,2.8.6.v20201231-1626] + com.google.gson.source [2.8.6.v20201231-1626,2.8.6.v20201231-1626] + com.jcraft.jsch [0.1.55.v20190404-1902,0.1.55.v20190404-1902] + com.jcraft.jsch.source [0.1.55.v20190404-1902,0.1.55.v20190404-1902] + com.jcraft.jzlib [1.1.1.v201205102305,1.1.1.v201205102305] + com.jcraft.jzlib.source [1.1.1.v201205102305,1.1.1.v201205102305] + javaewah [1.1.7.v20200107-0831,1.1.7.v20200107-0831] + javaewah.source [1.1.7.v20200107-0831,1.1.7.v20200107-0831] + javax.servlet [3.1.0.v201410161800,3.1.0.v201410161800] + javax.servlet.source [3.1.0.v201410161800,3.1.0.v201410161800] + net.bytebuddy.byte-buddy [1.9.0.v20181107-1410,1.9.0.v20181107-1410] + net.bytebuddy.byte-buddy-agent [1.9.0.v20181106-1534,1.9.0.v20181106-1534] + net.bytebuddy.byte-buddy-agent.source [1.9.0.v20181106-1534,1.9.0.v20181106-1534] + net.bytebuddy.byte-buddy.source [1.9.0.v20181107-1410,1.9.0.v20181107-1410] + net.i2p.crypto.eddsa [0.3.0.v20181102-1323,0.3.0.v20181102-1323] + net.i2p.crypto.eddsa.source [0.3.0.v20181102-1323,0.3.0.v20181102-1323] + org.apache.ant [1.10.9.v20201106-1946,1.10.9.v20201106-1946] + org.apache.ant.source [1.10.9.v20201106-1946,1.10.9.v20201106-1946] + org.apache.commons.codec [1.14.0.v20200818-1422,1.14.0.v20200818-1422] + org.apache.commons.codec.source [1.14.0.v20200818-1422,1.14.0.v20200818-1422] + org.apache.commons.compress [1.19.0.v20200106-2343,1.19.0.v20200106-2343] + org.apache.commons.compress.source [1.19.0.v20200106-2343,1.19.0.v20200106-2343] + org.apache.commons.logging [1.2.0.v20180409-1502,1.2.0.v20180409-1502] + org.apache.commons.logging.source [1.2.0.v20180409-1502,1.2.0.v20180409-1502] + org.apache.httpcomponents.httpclient [4.5.13.v20210128-2225,4.5.13.v20210128-2225] + org.apache.httpcomponents.httpclient.source [4.5.13.v20210128-2225,4.5.13.v20210128-2225] + org.apache.httpcomponents.httpcore [4.4.14.v20210128-2225,4.4.14.v20210128-2225] + org.apache.httpcomponents.httpcore.source [4.4.14.v20210128-2225,4.4.14.v20210128-2225] + org.apache.log4j [1.2.15.v201012070815,1.2.15.v201012070815] + org.apache.log4j.source [1.2.15.v201012070815,1.2.15.v201012070815] + org.apache.sshd.osgi [2.6.0.v20210201-2003,2.6.0.v20210201-2003] + org.apache.sshd.osgi.source [2.6.0.v20210201-2003,2.6.0.v20210201-2003] + org.apache.sshd.sftp [2.6.0.v20210201-2003,2.6.0.v20210201-2003] + org.apache.sshd.sftp.source [2.6.0.v20210201-2003,2.6.0.v20210201-2003] + org.assertj [3.14.0.v20200120-1926,3.14.0.v20200120-1926] + org.assertj.source [3.14.0.v20200120-1926,3.14.0.v20200120-1926] + org.bouncycastle.bcpg [1.65.0.v20200527-1955,1.65.0.v20200527-1955] + org.bouncycastle.bcpg.source [1.65.0.v20200527-1955,1.65.0.v20200527-1955] + org.bouncycastle.bcpkix [1.65.0.v20200527-1955,1.65.0.v20200527-1955] + org.bouncycastle.bcpkix.source [1.65.0.v20200527-1955,1.65.0.v20200527-1955] + org.bouncycastle.bcprov [1.65.1.v20200529-1514,1.65.1.v20200529-1514] + org.bouncycastle.bcprov.source [1.65.1.v20200529-1514,1.65.1.v20200529-1514] + org.hamcrest [1.1.0.v20090501071000,1.1.0.v20090501071000] + org.hamcrest.core [1.3.0.v20180420-1519,1.3.0.v20180420-1519] + org.hamcrest.core.source [1.3.0.v20180420-1519,1.3.0.v20180420-1519] + org.hamcrest.library [1.3.0.v20180524-2246,1.3.0.v20180524-2246] + org.hamcrest.library.source [1.3.0.v20180524-2246,1.3.0.v20180524-2246] + org.junit [4.13.0.v20200204-1500,4.13.0.v20200204-1500] + org.junit.source [4.13.0.v20200204-1500,4.13.0.v20200204-1500] + org.kohsuke.args4j [2.33.0.v20160323-2218,2.33.0.v20160323-2218] + org.kohsuke.args4j.source [2.33.0.v20160323-2218,2.33.0.v20160323-2218] + org.mockito [2.23.0.v20200310-1642,2.23.0.v20200310-1642] + org.mockito.source [2.23.0.v20200310-1642,2.23.0.v20200310-1642] + org.objenesis [2.6.0.v20180420-1519,2.6.0.v20180420-1519] + org.objenesis.source [2.6.0.v20180420-1519,2.6.0.v20180420-1519] + org.slf4j.api [1.7.30.v20200204-2150,1.7.30.v20200204-2150] + org.slf4j.api.source [1.7.30.v20200204-2150,1.7.30.v20200204-2150] + org.slf4j.binding.log4j12 [1.7.30.v20201108-2042,1.7.30.v20201108-2042] + org.slf4j.binding.log4j12.source [1.7.30.v20201108-2042,1.7.30.v20201108-2042] + org.tukaani.xz [1.8.0.v20180207-1613,1.8.0.v20180207-1613] + org.tukaani.xz.source [1.8.0.v20180207-1613,1.8.0.v20180207-1613] +} diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd index 70c426c18..4eec8aa35 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd @@ -1,20 +1,22 @@ target "jetty-9.4.x" with source configurePhase -location jetty-9.4.30 "https://download.eclipse.org/jetty/updates/jetty-bundles-9.x/9.4.30.v20200611/" { - org.eclipse.jetty.client [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.client.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.continuation [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.continuation.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.http [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.http.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.io [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.io.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.security [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.security.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.server [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.server.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.servlet [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.servlet.source [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.util [9.4.30.v20200611,9.4.30.v20200611] - org.eclipse.jetty.util.source [9.4.30.v20200611,9.4.30.v20200611] +location jetty-9.4.36 "https://download.eclipse.org/jetty/updates/jetty-bundles-9.x/9.4.36.v20210114/" { + org.eclipse.jetty.client [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.client.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.continuation [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.continuation.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.http [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.http.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.io [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.io.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.security [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.security.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.server [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.server.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.servlet [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.servlet.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.util [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.util.source [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.util.ajax [9.4.36.v20210114,9.4.36.v20210114] + org.eclipse.jetty.util.ajax.source [9.4.36.v20210114,9.4.36.v20210114] } diff --git a/org.eclipse.jgit.packaging/pom.xml b/org.eclipse.jgit.packaging/pom.xml index 165854d21..313c6632c 100644 --- a/org.eclipse.jgit.packaging/pom.xml +++ b/org.eclipse.jgit.packaging/pom.xml @@ -165,7 +165,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M1 + 3.0.0-M3 enforce-maven @@ -175,7 +175,7 @@ - 3.5.2 + 3.6.3 @@ -294,12 +294,12 @@ org.eclipse.cbi.maven.plugins eclipse-jarsigner-plugin - 1.1.5 + 1.1.7 org.codehaus.mojo build-helper-maven-plugin - 3.0.0 + 3.2.0 maven-clean-plugin @@ -318,7 +318,7 @@ org.apache.maven.plugins maven-site-plugin - 3.8.2 + 3.9.1 diff --git a/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..b853c6a7e 100644 --- a/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java index 2f09b7f12..4cbd61c69 100644 --- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/CloneTest.java @@ -11,7 +11,9 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.io.File; @@ -25,6 +27,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.transport.RefSpec; @@ -64,6 +67,45 @@ public void testClone() throws Exception { assertEquals("expected 1 branch", 1, branches.size()); } + @Test + public void testCloneInitialBranch() throws Exception { + createInitialCommit(); + + File gitDir = db.getDirectory(); + String sourceURI = gitDir.toURI().toString(); + File target = createTempDirectory("target"); + String cmd = "git clone --branch master " + sourceURI + " " + + shellQuote(target.getPath()); + String[] result = execute(cmd); + assertArrayEquals(new String[] { + "Cloning into '" + target.getPath() + "'...", "", "" }, result); + + Git git2 = Git.open(target); + List branches = git2.branchList().call(); + assertEquals("expected 1 branch", 1, branches.size()); + + Repository db2 = git2.getRepository(); + ObjectId head = db2.resolve("HEAD"); + assertNotNull(head); + assertNotEquals(ObjectId.zeroId(), head); + ObjectId master = db2.resolve("master"); + assertEquals(head, master); + } + + @Test + public void testCloneInitialBranchMissing() throws Exception { + createInitialCommit(); + + File gitDir = db.getDirectory(); + String sourceURI = gitDir.toURI().toString(); + File target = createTempDirectory("target"); + String cmd = "git clone --branch foo " + sourceURI + " " + + shellQuote(target.getPath()); + Die e = assertThrows(Die.class, () -> execute(cmd)); + assertEquals("Remote branch 'foo' not found in upstream origin", + e.getMessage()); + } + private RevCommit createInitialCommit() throws Exception { JGitTestUtil.writeTrashFile(db, "hello.txt", "world"); git.add().addFilepattern("hello.txt").call(); diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/InitTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/InitTest.java index 84474e33c..88789d338 100644 --- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/InitTest.java +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/InitTest.java @@ -11,11 +11,14 @@ package org.eclipse.jgit.pgm; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import java.io.File; +import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.CLIRepositoryTestCase; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Repository; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -54,4 +57,22 @@ public void testInitDirectory() throws Exception { assertArrayEquals(expecteds, result); } + @Test + public void testInitDirectoryInitialBranch() throws Exception { + File workDirectory = tempFolder.getRoot(); + File gitDirectory = new File(workDirectory, Constants.DOT_GIT); + + String[] result = execute( + "git init -b main '" + workDirectory.getCanonicalPath() + "'"); + + String[] expecteds = new String[] { + "Initialized empty Git repository in " + + gitDirectory.getCanonicalPath(), + "" }; + assertArrayEquals(expecteds, result); + + try (Repository repo = new FileRepository(gitDirectory)) { + assertEquals("refs/heads/main", repo.getFullBranch()); + } + } } diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF index 6822784f7..c56224e3a 100644 --- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF @@ -8,6 +8,7 @@ Bundle-Vendor: %Bundle-Vendor Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: javax.servlet;version="[3.1.0,4.0.0)", + org.apache.commons.logging;version="[1.2,2.0)", org.eclipse.jetty.server;version="[9.4.5,10.0.0)", org.eclipse.jetty.server.handler;version="[9.4.5,10.0.0)", org.eclipse.jetty.servlet;version="[9.4.5,10.0.0)", @@ -22,12 +23,10 @@ Import-Package: javax.servlet;version="[3.1.0,4.0.0)", org.eclipse.jgit.dircache;version="[6.0.0,6.1.0)", org.eclipse.jgit.errors;version="[6.0.0,6.1.0)", org.eclipse.jgit.gitrepo;version="[6.0.0,6.1.0)", - org.eclipse.jgit.internal.ketch;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.file;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.io;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.pack;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.reftable;version="[6.0.0,6.1.0)", - org.eclipse.jgit.internal.storage.reftree;version="[6.0.0,6.1.0)", org.eclipse.jgit.lfs;version="[6.0.0,6.1.0)", org.eclipse.jgit.lfs.server;version="[6.0.0,6.1.0)", org.eclipse.jgit.lfs.server.fs;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin index 062b9643a..e645255e9 100644 --- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin +++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin @@ -47,7 +47,6 @@ org.eclipse.jgit.pgm.debug.MakeCacheTree org.eclipse.jgit.pgm.debug.ReadDirCache org.eclipse.jgit.pgm.debug.ReadReftable org.eclipse.jgit.pgm.debug.RebuildCommitGraph -org.eclipse.jgit.pgm.debug.RebuildRefTree org.eclipse.jgit.pgm.debug.ShowCacheTree org.eclipse.jgit.pgm.debug.ShowCommands org.eclipse.jgit.pgm.debug.ShowDirCache diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index 6112a272e..83846ee8e 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -77,14 +77,15 @@ invalidHttpProxyOnlyHttpSupported=Invalid http_proxy: {0}: Only http supported. invalidRecurseSubmodulesMode=Invalid recurse submodules mode: {0} invalidUntrackedFilesMode=Invalid untracked files mode ''{0}'' jgitVersion=jgit version {0} -lineFormat={0} -listeningOn=Listening on {0} lfsNoAccessKey=No accessKey in {0} lfsNoSecretKey=No secretKey in {0} lfsProtocolUrl=LFS protocol URL: {0} lfsStoreDirectory=LFS objects stored in: {0} lfsStoreUrl=LFS store URL: {0} lfsUnknownStoreType="Unknown LFS store type: {0}" +lineFormat={0} +listeningOn=Listening on {0} +logNoSignatureVerifier="No signature verifier available" mergeConflict=CONFLICT(content): Merge conflict in {0} mergeCheckoutConflict=error: Your local changes to the following files would be overwritten by merge: mergeFailed=Automatic merge failed; fix conflicts and then commit the result @@ -118,7 +119,6 @@ metaVar_file=FILE metaVar_filepattern=filepattern metaVar_gitDir=GIT_DIR metaVar_hostName=HOSTNAME -metaVar_ketchServerType=SERVERTYPE metaVar_lfsStorage=STORAGE metaVar_linesOfContext=lines metaVar_message=message @@ -143,6 +143,7 @@ metaVar_s3Region=REGION metaVar_s3StorageClass=STORAGE-CLASS metaVar_seconds=SECONDS metaVar_service=SERVICE +metaVar_tagLocalUser= metaVar_treeish=tree-ish metaVar_uriish=uri-ish metaVar_url=URL @@ -246,7 +247,6 @@ usage_DisplayTheVersionOfJgit=Display the version of jgit usage_Gc=Cleanup unnecessary files and optimize the local repository usage_Glog=View commit history as a graph usage_IndexPack=Build pack index file for an existing packed archive -usage_ketchServerType=Ketch server type usage_LFSDirectory=Directory to store large objects usage_LFSPort=Server http port usage_LFSRunStore=Store (fs | s3), store lfs objects in file system or Amazon S3 @@ -266,8 +266,6 @@ usage_PreserveOldPacks=Preserve old pack files by moving them into the preserved usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files usage_ReadDirCache= Read the DirCache 100 times usage_RebuildCommitGraph=Recreate a repository from another one's commit graph -usage_RebuildRefTree=Copy references into a RefTree -usage_RebuildRefTreeEnable=set extensions.refStorage = reftree usage_Remote=Manage set of tracked repositories usage_RepositoryToReadFrom=Repository to read from usage_RepositoryToReceiveInto=Repository to receive into @@ -414,6 +412,7 @@ usage_show=Display one commit usage_showRefNamesMatchingCommits=Show ref names matching commits usage_showPatch=display patch usage_showNotes=Add this ref to the list of note branches from which notes are displayed +usage_showSignature=Verify signatures of signed commits in the log usage_showTimeInMilliseconds=Show mtime in milliseconds usage_squash=Squash commits as if a real merge happened, but do not make a commit or move the HEAD. usage_srcPrefix=show the source prefix instead of "a/" @@ -421,13 +420,19 @@ usage_sshDriver=Selects the built-in ssh library to use, JSch or Apache MINA ssh usage_symbolicVersionForTheProject=Symbolic version for the project usage_tags=fetch all tags usage_notags=do not fetch tags +usage_tagAnnotated=create an annotated tag, unsigned unless -s or -u are given, or config tag.gpgSign is true usage_tagDelete=delete tag -usage_tagMessage=tag message +usage_tagLocalUser=create a signed annotated tag using the specified GPG key ID +usage_tagMessage=create an annotated tag with the given message, unsigned unless -s or -u are given, or config tag.gpgSign is true, or tar.forceSignAnnotated is true and -a is not given +usage_tagSign=create a signed annotated tag +usage_tagNoSign=suppress signing the tag +usage_tagVerify=Verify the GPG signature usage_untrackedFilesMode=show untracked files usage_updateRef=reference to update usage_updateRemoteRefsFromAnotherRepository=Update remote refs from another repository usage_useNameInsteadOfOriginToTrackUpstream=use instead of 'origin' to track upstream usage_checkoutBranchAfterClone=check out named branch instead of remote's HEAD +usage_initialBranch=initial branch of the newly created repository (default 'master', can be configured via config option init.defaultBranch) usage_viewCommitHistory=View commit history usage_orphan=Create a new orphan branch. The first commit made on this new branch will have no parents and it will be the root of a new history totally disconnected from other branches and commits. usernameFor=Username for {0}: diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java index 8f80d6d70..f28915d3f 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Clone.java @@ -18,6 +18,7 @@ import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.InvalidRemoteException; +import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.TextProgressMonitor; @@ -50,6 +51,9 @@ class Clone extends AbstractFetchCommand implements CloneCommand.Callback { @Option(name = "--recurse-submodules", usage = "usage_recurseSubmodules") private boolean cloneSubmodules; + @Option(name = "--timeout", metaVar = "metaVar_seconds", usage = "usage_abortConnectionIfNoActivity") + int timeout = -1; + @Argument(index = 0, required = true, metaVar = "metaVar_uriish") private String sourceUri; @@ -90,9 +94,8 @@ protected void run() throws Exception { CloneCommand command = Git.cloneRepository(); command.setURI(sourceUri).setRemote(remoteName).setBare(isBare) - .setMirror(isMirror) - .setNoCheckout(noCheckout).setBranch(branch) - .setCloneSubmodules(cloneSubmodules); + .setMirror(isMirror).setNoCheckout(noCheckout).setBranch(branch) + .setCloneSubmodules(cloneSubmodules).setTimeout(timeout); command.setGitDir(gitdir == null ? null : new File(gitdir)); command.setDirectory(localNameF); @@ -108,6 +111,8 @@ protected void run() throws Exception { db = command.call().getRepository(); if (msgs && db.resolve(Constants.HEAD) == null) outw.println(CLIText.get().clonedEmptyRepository); + } catch (TransportException e) { + throw die(e.getMessage(), e); } catch (InvalidRemoteException e) { throw die(MessageFormat.format(CLIText.get().doesNotExist, sourceUri), e); diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java index bf9102552..f987f2c80 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java @@ -13,19 +13,12 @@ import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URISyntaxException; import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; import org.eclipse.jgit.errors.ConfigInvalidException; -import org.eclipse.jgit.internal.ketch.KetchLeader; -import org.eclipse.jgit.internal.ketch.KetchLeaderCache; -import org.eclipse.jgit.internal.ketch.KetchPreReceive; -import org.eclipse.jgit.internal.ketch.KetchSystem; -import org.eclipse.jgit.internal.ketch.KetchText; -import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -33,10 +26,7 @@ import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.DaemonClient; import org.eclipse.jgit.transport.DaemonService; -import org.eclipse.jgit.transport.ReceivePack; import org.eclipse.jgit.transport.resolver.FileResolver; -import org.eclipse.jgit.transport.resolver.ReceivePackFactory; -import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.SystemReader; import org.kohsuke.args4j.Argument; @@ -71,13 +61,6 @@ class Daemon extends TextBuiltin { @Option(name = "--export-all", usage = "usage_exportWithoutGitDaemonExportOk") boolean exportAll; - @Option(name = "--ketch", metaVar = "metaVar_ketchServerType", usage = "usage_ketchServerType") - KetchServerType ketchServerType; - - enum KetchServerType { - LEADER; - } - @Argument(required = true, metaVar = "metaVar_directory", usage = "usage_directoriesToExport") List directory = new ArrayList<>(); @@ -102,9 +85,9 @@ protected void run() throws Exception { } cfg = new FileBasedConfig(configFile, FS.DETECTED); } - cfg.load(); - new WindowCacheConfig().fromConfig(cfg).install(); - packConfig.fromConfig(cfg); + cfg.load(); + new WindowCacheConfig().fromConfig(cfg).install(); + packConfig.fromConfig(cfg); int threads = packConfig.getThreads(); if (threads <= 0) @@ -137,9 +120,6 @@ protected void run() throws Exception { service(d, n).setOverridable(true); for (String n : forbidOverride) service(d, n).setOverridable(false); - if (ketchServerType == KetchServerType.LEADER) { - startKetchLeader(d); - } d.start(); outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress())); } @@ -162,24 +142,4 @@ private static DaemonService service( throw die(MessageFormat.format(CLIText.get().serviceNotSupported, n)); return svc; } - - private void startKetchLeader(org.eclipse.jgit.transport.Daemon daemon) { - KetchSystem system = new KetchSystem(); - final KetchLeaderCache leaders = new KetchLeaderCache(system); - final ReceivePackFactory factory; - - factory = daemon.getReceivePackFactory(); - daemon.setReceivePackFactory((DaemonClient req, Repository repo) -> { - ReceivePack rp = factory.create(req, repo); - KetchLeader leader; - try { - leader = leaders.get(repo); - } catch (URISyntaxException err) { - throw new ServiceNotEnabledException( - KetchText.get().invalidFollowerUri, err); - } - rp.setPreReceiveHook(new KetchPreReceive(leader)); - return rp; - }); - } } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Init.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Init.java index 7f59ef43d..7a0d96d41 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Init.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Init.java @@ -24,6 +24,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.pgm.internal.CLIText; +import org.eclipse.jgit.util.StringUtils; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Option; @@ -32,6 +33,10 @@ class Init extends TextBuiltin { @Option(name = "--bare", usage = "usage_CreateABareRepository") private boolean bare; + @Option(name = "--initial-branch", aliases = { "-b" }, + metaVar = "metaVar_branchName", usage = "usage_initialBranch") + private String branch; + @Argument(index = 0, metaVar = "metaVar_directory") private String directory; @@ -54,6 +59,9 @@ protected void run() { } Repository repository; try { + if (!StringUtils.isEmptyOrNull(branch)) { + command.setInitialBranch(branch); + } repository = command.call().getRepository(); outw.println(MessageFormat.format( CLIText.get().initializedEmptyGitRepositoryIn, diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java index 55efd23c6..353b64b9b 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2010, Google Inc. - * Copyright (C) 2006-2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2006, 2008, Robin Rosenberg + * Copyright (C) 2008, 2021, Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -31,12 +31,17 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.notes.NoteMap; import org.eclipse.jgit.pgm.internal.CLIText; +import org.eclipse.jgit.pgm.internal.VerificationUtils; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.util.GitDateFormatter; @@ -68,6 +73,9 @@ void addAdditionalNoteRef(String notesRef) { additionalNoteRefs.add(notesRef); } + @Option(name = "--show-signature", usage = "usage_showSignature") + private boolean showSignature; + @Option(name = "--date", usage = "usage_date") void dateFormat(String date) { if (date.toLowerCase(Locale.ROOT).equals(date)) @@ -147,6 +155,10 @@ void noPrefix(@SuppressWarnings("unused") boolean on) { // END -- Options shared with Diff + private GpgSignatureVerifier verifier; + + private GpgConfig config; + Log() { dateFormatter = new GitDateFormatter(Format.DEFAULT); } @@ -161,6 +173,7 @@ protected void init(Repository repository, String gitDir) { /** {@inheritDoc} */ @Override protected void run() { + config = new GpgConfig(db.getConfig()); diffFmt.setRepository(db); try { diffFmt.setPathFilter(pathFilter); @@ -197,6 +210,9 @@ protected void run() { throw die(e.getMessage(), e); } finally { diffFmt.close(); + if (verifier != null) { + verifier.clear(); + } } } @@ -229,6 +245,9 @@ protected void show(RevCommit c) throws Exception { } outw.println(); + if (showSignature) { + showSignature(c); + } final PersonIdent author = c.getAuthorIdent(); outw.println(MessageFormat.format(CLIText.get().authorInfo, author.getName(), author.getEmailAddress())); outw.println(MessageFormat.format(CLIText.get().dateInfo, @@ -252,6 +271,27 @@ protected void show(RevCommit c) throws Exception { outw.flush(); } + private void showSignature(RevCommit c) throws IOException { + if (c.getRawGpgSignature() == null) { + return; + } + if (verifier == null) { + GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory + .getDefault(); + if (factory == null) { + throw die(CLIText.get().logNoSignatureVerifier, null); + } + verifier = factory.getVerifier(); + } + SignatureVerification verification = verifier.verifySignature(c, + config); + if (verification == null) { + return; + } + VerificationUtils.writeVerification(outw, verification, + verifier.getName(), c.getCommitterIdent()); + } + /** * @param c * @return true if at least one note was printed, diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/LsRemote.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/LsRemote.java index 055b48a15..83446ccd5 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/LsRemote.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/LsRemote.java @@ -79,7 +79,7 @@ private void show(AnyObjectId id, String name) private void show(Ref ref, String name) throws IOException { - outw.print("ref: "); + outw.print("ref: "); //$NON-NLS-1$ outw.print(ref.getName()); outw.print('\t'); outw.print(name); diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java index 5f9551e52..3beab60a8 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java @@ -29,10 +29,15 @@ import org.eclipse.jgit.errors.RevisionSyntaxException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; import org.eclipse.jgit.pgm.internal.CLIText; +import org.eclipse.jgit.pgm.internal.VerificationUtils; import org.eclipse.jgit.pgm.opt.PathTreeFilterHandler; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; @@ -41,6 +46,7 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.eclipse.jgit.util.RawParseUtils; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Option; @@ -58,6 +64,9 @@ class Show extends TextBuiltin { @Option(name = "--", metaVar = "metaVar_path", handler = PathTreeFilterHandler.class) protected TreeFilter pathFilter = TreeFilter.ALL; + @Option(name = "--show-signature", usage = "usage_showSignature") + private boolean showSignature; + // BEGIN -- Options shared with Diff @Option(name = "-p", usage = "usage_showPatch") boolean showPatch; @@ -219,13 +228,20 @@ private void show(RevTag tag) throws IOException { } outw.println(); - final String[] lines = tag.getFullMessage().split("\n"); //$NON-NLS-1$ - for (String s : lines) { - outw.print(" "); //$NON-NLS-1$ - outw.print(s); - outw.println(); + String fullMessage = tag.getFullMessage(); + if (!fullMessage.isEmpty()) { + String[] lines = tag.getFullMessage().split("\n"); //$NON-NLS-1$ + for (String s : lines) { + outw.println(s); + } + } + byte[] rawSignature = tag.getRawGpgSignature(); + if (rawSignature != null) { + String[] lines = RawParseUtils.decode(rawSignature).split("\n"); //$NON-NLS-1$ + for (String s : lines) { + outw.println(s); + } } - outw.println(); } @@ -253,6 +269,10 @@ private void show(RevWalk rw, RevCommit c) throws IOException { c.getId().copyTo(outbuffer, outw); outw.println(); + if (showSignature) { + showSignature(c); + } + final PersonIdent author = c.getAuthorIdent(); outw.println(MessageFormat.format(CLIText.get().authorInfo, author.getName(), author.getEmailAddress())); @@ -291,4 +311,28 @@ private void showDiff(RevCommit c) throws IOException { } outw.println(); } + + private void showSignature(RevCommit c) throws IOException { + if (c.getRawGpgSignature() == null) { + return; + } + GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory + .getDefault(); + if (factory == null) { + throw die(CLIText.get().logNoSignatureVerifier, null); + } + GpgSignatureVerifier verifier = factory.getVerifier(); + GpgConfig config = new GpgConfig(db.getConfig()); + try { + SignatureVerification verification = verifier.verifySignature(c, + config); + if (verification == null) { + return; + } + VerificationUtils.writeVerification(outw, verification, + verifier.getName(), c.getCommitterIdent()); + } finally { + verifier.clear(); + } + } } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java index b408b78f3..e2cd31d19 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java @@ -4,7 +4,7 @@ * Copyright (C) 2008, Charles O'Farrell * Copyright (C) 2008, Robin Rosenberg * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2021 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -22,26 +22,59 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ListTagCommand; import org.eclipse.jgit.api.TagCommand; +import org.eclipse.jgit.api.VerificationResult; +import org.eclipse.jgit.api.VerifySignatureCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.pgm.internal.CLIText; +import org.eclipse.jgit.pgm.internal.VerificationUtils; +import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevWalk; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Option; @Command(common = true, usage = "usage_CreateATag") class Tag extends TextBuiltin { - @Option(name = "-f", usage = "usage_forceReplacingAnExistingTag") + + @Option(name = "--force", aliases = { "-f" }, forbids = { "--delete", + "--verify" }, usage = "usage_forceReplacingAnExistingTag") private boolean force; - @Option(name = "-d", usage = "usage_tagDelete") + @Option(name = "--delete", aliases = { "-d" }, forbids = { + "--verify" }, usage = "usage_tagDelete") private boolean delete; - @Option(name = "-m", metaVar = "metaVar_message", usage = "usage_tagMessage") - private String message = ""; //$NON-NLS-1$ + @Option(name = "--annotate", aliases = { + "-a" }, forbids = { "--delete", + "--verify" }, usage = "usage_tagAnnotated") + private boolean annotated; + + @Option(name = "-m", forbids = { "--delete", + "--verify" }, metaVar = "metaVar_message", usage = "usage_tagMessage") + private String message; + + @Option(name = "--sign", aliases = { "-s" }, forbids = { + "--no-sign", "--delete", "--verify" }, usage = "usage_tagSign") + private boolean sign; + + @Option(name = "--no-sign", usage = "usage_tagNoSign", forbids = { + "--sign", "--delete", "--verify" }) + private boolean noSign; + + @Option(name = "--local-user", aliases = { + "-u" }, forbids = { "--delete", + "--verify" }, metaVar = "metaVar_tagLocalUser", usage = "usage_tagLocalUser") + private String gpgKeyId; + + @Option(name = "--verify", aliases = { "-v" }, forbids = { "--delete", + "--force", "--annotate", "-m", "--sign", "--no-sign", + "--local-user" }, usage = "usage_tagVerify") + private boolean verify; @Argument(index = 0, metaVar = "metaVar_name") private String tagName; @@ -54,7 +87,25 @@ class Tag extends TextBuiltin { protected void run() { try (Git git = new Git(db)) { if (tagName != null) { - if (delete) { + if (verify) { + VerifySignatureCommand verifySig = git.verifySignature() + .setMode(VerifySignatureCommand.VerifyMode.TAGS) + .addName(tagName); + + VerificationResult verification = verifySig.call() + .get(tagName); + if (verification == null) { + showUnsigned(git, tagName); + } else { + Throwable error = verification.getException(); + if (error != null) { + throw die(error.getMessage(), error); + } + writeVerification(verifySig.getVerifier().getName(), + (RevTag) verification.getObject(), + verification.getVerification()); + } + } else if (delete) { List deletedTags = git.tagDelete().setTags(tagName) .call(); if (deletedTags.isEmpty()) { @@ -70,6 +121,18 @@ protected void run() { command.setObjectId(walk.parseAny(object)); } } + if (noSign) { + command.setSigned(false); + } else if (sign) { + command.setSigned(true); + } + if (annotated) { + command.setAnnotated(true); + } else if (message == null && !sign && gpgKeyId == null) { + // None of -a, -m, -s, -u given + command.setAnnotated(false); + } + command.setSigningKey(gpgKeyId); try { command.call(); } catch (RefAlreadyExistsException e) { @@ -88,4 +151,36 @@ protected void run() { throw die(e.getMessage(), e); } } + + private void showUnsigned(Git git, String wantedTag) throws IOException { + ObjectId id = git.getRepository().resolve(wantedTag); + if (id != null && !ObjectId.zeroId().equals(id)) { + try (RevWalk walk = new RevWalk(git.getRepository())) { + showTag(walk.parseTag(id)); + } + } else { + throw die( + MessageFormat.format(CLIText.get().tagNotFound, wantedTag)); + } + } + + private void showTag(RevTag tag) throws IOException { + outw.println("object " + tag.getObject().name()); //$NON-NLS-1$ + outw.println("type " + Constants.typeString(tag.getObject().getType())); //$NON-NLS-1$ + outw.println("tag " + tag.getTagName()); //$NON-NLS-1$ + outw.println("tagger " + tag.getTaggerIdent().toExternalString()); //$NON-NLS-1$ + outw.println(); + outw.print(tag.getFullMessage()); + } + + private void writeVerification(String name, RevTag tag, + SignatureVerification verification) throws IOException { + showTag(tag); + if (verification == null) { + outw.println(); + return; + } + VerificationUtils.writeVerification(outw, verification, name, + tag.getTaggerIdent()); + } } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/TextBuiltin.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/TextBuiltin.java index 0b02dd148..f70e72d43 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/TextBuiltin.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/TextBuiltin.java @@ -63,7 +63,7 @@ public abstract class TextBuiltin { private boolean help; @Option(name = "--ssh", usage = "usage_sshDriver") - private SshDriver sshDriver = SshDriver.JSCH; + private SshDriver sshDriver = SshDriver.APACHE; /** * Input stream, typically this is standard input. @@ -220,7 +220,7 @@ public final void execute(String[] args) throws Exception { SshdSessionFactory factory = new SshdSessionFactory( new JGitKeyCache(), new DefaultProxyDataFactory()); Runtime.getRuntime() - .addShutdownHook(new Thread(() -> factory.close())); + .addShutdownHook(new Thread(factory::close)); SshSessionFactory.setInstance(factory); break; } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java index 630fac549..f23f4cf0e 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java @@ -23,7 +23,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.eclipse.jgit.internal.storage.file.FileReftableStack; import org.eclipse.jgit.internal.storage.io.BlockSource; @@ -47,6 +49,7 @@ enum Test { SEEK_COLD, SEEK_HOT, BY_ID_COLD, BY_ID_HOT, WRITE_STACK, + GET_REFS_EXCLUDING_REF } @Option(name = "--tries") @@ -91,7 +94,11 @@ protected void run() throws Exception { case WRITE_STACK: writeStack(); break; - } + case GET_REFS_EXCLUDING_REF : + getRefsExcludingWithSeekPast(ref); + getRefsExcludingWithFilter(ref); + break; + } } private void printf(String fmt, Object... args) throws IOException { @@ -315,4 +322,49 @@ private void byIdHot(ObjectId id) throws Exception { printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable", tot / 1000, (((double) tot) / tries) / 1000, tries); } + + @SuppressWarnings({"nls", "boxing"}) + private void getRefsExcludingWithFilter(String prefix) throws Exception { + long startTime = System.nanoTime(); + List allRefs = new ArrayList<>(); + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + try (RefCursor rc = reader.allRefs()) { + while (rc.next()) { + allRefs.add(rc.getRef()); + } + } + } + int total = allRefs.size(); + allRefs = allRefs.stream().filter(r -> r.getName().startsWith(prefix)).collect(Collectors.toList()); + int notStartWithPrefix = allRefs.size(); + int startWithPrefix = total - notStartWithPrefix; + long totalTime = System.nanoTime() - startTime; + printf("total time the action took using filter: %10d usec", totalTime / 1000); + printf("number of refs that start with prefix: %d", startWithPrefix); + printf("number of refs that don't start with prefix: %d", notStartWithPrefix); + } + + @SuppressWarnings({"nls", "boxing"}) + private void getRefsExcludingWithSeekPast(String prefix) throws Exception { + long start = System.nanoTime(); + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + try (RefCursor rc = reader.allRefs()) { + while (rc.next()) { + if (rc.getRef().getName().startsWith(prefix)) { + break; + } + } + rc.seekPastPrefix(prefix); + while (rc.next()) { + rc.getRef(); + } + } + } + long tot = System.nanoTime() - start; + printf("total time the action took using seek: %10d usec", tot / 1000); + } } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/RebuildRefTree.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/RebuildRefTree.java deleted file mode 100644 index 38951ba42..000000000 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/RebuildRefTree.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2015, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.pgm.debug; - -import static org.eclipse.jgit.lib.Constants.HEAD; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ConfigConstants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.StoredConfig; -import org.eclipse.jgit.pgm.Command; -import org.eclipse.jgit.pgm.TextBuiltin; -import org.eclipse.jgit.revwalk.RevWalk; -import org.kohsuke.args4j.Option; - -@Command(usage = "usage_RebuildRefTree") -class RebuildRefTree extends TextBuiltin { - @Option(name = "--enable", usage = "usage_RebuildRefTreeEnable") - boolean enable; - - private String txnNamespace; - private String txnCommitted; - - /** {@inheritDoc} */ - @Override - protected void run() throws Exception { - try (ObjectReader reader = db.newObjectReader(); - RevWalk rw = new RevWalk(reader); - ObjectInserter inserter = db.newObjectInserter()) { - RefDatabase refDb = db.getRefDatabase(); - if (refDb instanceof RefTreeDatabase) { - RefTreeDatabase d = (RefTreeDatabase) refDb; - refDb = d.getBootstrap(); - txnNamespace = d.getTxnNamespace(); - txnCommitted = d.getTxnCommitted(); - } else { - RefTreeDatabase d = new RefTreeDatabase(db, refDb); - txnNamespace = d.getTxnNamespace(); - txnCommitted = d.getTxnCommitted(); - } - - errw.format("Rebuilding %s from %s", //$NON-NLS-1$ - txnCommitted, refDb.getClass().getSimpleName()); - errw.println(); - errw.flush(); - - CommitBuilder b = new CommitBuilder(); - Ref ref = refDb.exactRef(txnCommitted); - RefUpdate update = refDb.newUpdate(txnCommitted, true); - ObjectId oldTreeId; - - if (ref != null && ref.getObjectId() != null) { - ObjectId oldId = ref.getObjectId(); - update.setExpectedOldObjectId(oldId); - b.setParentId(oldId); - oldTreeId = rw.parseCommit(oldId).getTree(); - } else { - update.setExpectedOldObjectId(ObjectId.zeroId()); - oldTreeId = ObjectId.zeroId(); - } - - RefTree tree = rebuild(refDb); - b.setTreeId(tree.writeTree(inserter)); - b.setAuthor(new PersonIdent(db)); - b.setCommitter(b.getAuthor()); - if (b.getTreeId().equals(oldTreeId)) { - return; - } - - update.setNewObjectId(inserter.insert(b)); - inserter.flush(); - - RefUpdate.Result result = update.update(rw); - switch (result) { - case NEW: - case FAST_FORWARD: - break; - default: - throw die(String.format("%s: %s", update.getName(), result)); //$NON-NLS-1$ - } - - if (enable && !(db.getRefDatabase() instanceof RefTreeDatabase)) { - StoredConfig cfg = db.getConfig(); - cfg.setInt(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 1); - cfg.setString(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, - ConfigConstants.CONFIG_KEY_REFSTORAGE, - ConfigConstants.CONFIG_REFSTORAGE_REFTREE); - cfg.save(); - errw.println("Enabled reftree."); //$NON-NLS-1$ - errw.flush(); - } - } - } - - private RefTree rebuild(RefDatabase refdb) throws IOException { - RefTree tree = RefTree.newEmptyTree(); - List cmds - = new ArrayList<>(); - - Ref head = refdb.exactRef(HEAD); - if (head != null) { - cmds.add(new org.eclipse.jgit.internal.storage.reftree.Command( - null, - head)); - } - - for (Ref r : refdb.getRefs()) { - if (r.getName().equals(txnCommitted) || r.getName().equals(HEAD) - || r.getName().startsWith(txnNamespace)) { - continue; - } - cmds.add(new org.eclipse.jgit.internal.storage.reftree.Command( - null, - db.getRefDatabase().peel(r))); - } - tree.apply(cmds); - return tree; - } -} diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java index c68019e5d..991b3ba58 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, 2013 Sasa Zivkov - * Copyright (C) 2013, Obeo and others + * Copyright (C) 2013, 2021 Obeo and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -163,6 +163,7 @@ public static String fatalError(String message) { /***/ public String lfsUnknownStoreType; /***/ public String lineFormat; /***/ public String listeningOn; + /***/ public String logNoSignatureVerifier; /***/ public String mergeCheckoutConflict; /***/ public String mergeConflict; /***/ public String mergeFailed; diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java new file mode 100644 index 000000000..c1f8a86a8 --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.pgm.internal; + +import java.io.IOException; + +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.util.GitDateFormatter; +import org.eclipse.jgit.util.SignatureUtils; +import org.eclipse.jgit.util.io.ThrowingPrintWriter; + +/** + * Utilities for signature verification. + */ +public final class VerificationUtils { + + private VerificationUtils() { + // No instantiation + } + + /** + * Writes information about a signature verification to the given writer. + * + * @param out + * to write to + * @param verification + * to show + * @param name + * of the verifier used + * @param creator + * of the object verified; used for time zone information + * @throws IOException + * if writing fails + */ + public static void writeVerification(ThrowingPrintWriter out, + SignatureVerification verification, String name, + PersonIdent creator) throws IOException { + String[] text = SignatureUtils + .toString(verification, creator, + new GitDateFormatter(GitDateFormatter.Format.LOCALE)) + .split("\n"); //$NON-NLS-1$ + for (String line : text) { + out.print(name); + out.print(": "); //$NON-NLS-1$ + out.println(line); + } + } +} diff --git a/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs index 822846c4d..cba893f04 100644 --- a/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF index fe3a2616f..d881d8775 100644 --- a/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.ssh.apache.test/META-INF/MANIFEST.MF @@ -7,16 +7,18 @@ Bundle-Version: 6.0.0.qualifier Bundle-Vendor: %Bundle-Vendor Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.apache.sshd.client.config.hosts;version="[2.4.0,2.5.0)", - org.apache.sshd.common;version="[2.4.0,2.5.0)", - org.apache.sshd.common.auth;version="[2.4.0,2.5.0)", - org.apache.sshd.common.config.keys;version="[2.4.0,2.5.0)", - org.apache.sshd.common.keyprovider;version="[2.4.0,2.5.0)", - org.apache.sshd.common.session;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.net;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.security;version="[2.4.0,2.5.0)", - org.apache.sshd.server;version="[2.4.0,2.5.0)", - org.apache.sshd.server.forward;version="[2.4.0,2.5.0)", +Import-Package: org.apache.sshd.client.config.hosts;version="[2.6.0,2.7.0)", + org.apache.sshd.common;version="[2.6.0,2.7.0)", + org.apache.sshd.common.auth;version="[2.6.0,2.7.0)", + org.apache.sshd.common.config.keys;version="[2.6.0,2.7.0)", + org.apache.sshd.common.helpers;version="[2.6.0,2.7.0)", + org.apache.sshd.common.keyprovider;version="[2.6.0,2.7.0)", + org.apache.sshd.common.session;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.net;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.security;version="[2.6.0,2.7.0)", + org.apache.sshd.core;version="[2.6.0,2.7.0)", + org.apache.sshd.server;version="[2.6.0,2.7.0)", + org.apache.sshd.server.forward;version="[2.6.0,2.7.0)", org.eclipse.jgit.api;version="[6.0.0,6.1.0)", org.eclipse.jgit.api.errors;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.transport.sshd.proxy;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java new file mode 100644 index 000000000..0ad96b9ac --- /dev/null +++ b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshProtocol2Test.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport.sshd; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.Arrays; + +import org.eclipse.jgit.junit.ssh.SshBasicTestBase; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.transport.SshSessionFactory; +import org.eclipse.jgit.util.FS; + +public class ApacheSshProtocol2Test extends SshBasicTestBase { + + @Override + protected SshSessionFactory createSessionFactory() { + SshdSessionFactory result = new SshdSessionFactory(new JGitKeyCache(), + null); + // The home directory is mocked at this point! + result.setHomeDirectory(FS.DETECTED.userHome()); + result.setSshDirectory(sshDir); + return result; + } + + @Override + protected void installConfig(String... config) { + File configFile = new File(sshDir, Constants.CONFIG); + if (config != null) { + try { + Files.write(configFile.toPath(), Arrays.asList(config)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @Override + public void setUp() throws Exception { + super.setUp(); + StoredConfig config = ((Repository) db).getConfig(); + config.setInt("protocol", null, "version", 2); + config.save(); + } +} diff --git a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java index 3427da667..97f97f902 100644 --- a/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java +++ b/org.eclipse.jgit.ssh.apache.test/tst/org/eclipse/jgit/transport/sshd/ApacheSshTest.java @@ -9,6 +9,7 @@ */ package org.eclipse.jgit.transport.sshd; +import static org.apache.sshd.core.CoreModuleProperties.MAX_CONCURRENT_SESSIONS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -33,7 +34,6 @@ import org.apache.sshd.client.config.hosts.KnownHostEntry; import org.apache.sshd.client.config.hosts.KnownHostHashValue; -import org.apache.sshd.common.PropertyResolverUtils; import org.apache.sshd.common.config.keys.AuthorizedKeyEntry; import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.common.config.keys.PublicKeyEntry; @@ -41,7 +41,6 @@ import org.apache.sshd.common.session.Session; import org.apache.sshd.common.util.net.SshdSocketAddress; import org.apache.sshd.server.ServerAuthenticationManager; -import org.apache.sshd.server.ServerFactoryManager; import org.apache.sshd.server.SshServer; import org.apache.sshd.server.forward.StaticDecisionForwardingFilter; import org.eclipse.jgit.api.Git; @@ -216,8 +215,8 @@ public void testHugePreamble() throws Exception { */ @Test public void testCloneAndFetchWithSessionLimit() throws Exception { - PropertyResolverUtils.updateProperty(server.getPropertyResolver(), - ServerFactoryManager.MAX_CONCURRENT_SESSIONS, 2); + MAX_CONCURRENT_SESSIONS + .set(server.getPropertyResolver(), Integer.valueOf(2)); File localClone = cloneWith("ssh://localhost/doesntmatter", defaultCloneDir, null, // "Host localhost", // diff --git a/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF b/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF index 96b40ad15..faa52b26a 100644 --- a/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.ssh.apache/META-INF/MANIFEST.MF @@ -33,49 +33,51 @@ Export-Package: org.eclipse.jgit.internal.transport.sshd;version="6.0.0";x-inter org.apache.sshd.client.session, org.apache.sshd.client.keyverifier" Import-Package: net.i2p.crypto.eddsa;version="[0.3.0,0.4.0)", - org.apache.sshd.agent;version="[2.4.0,2.5.0)", - org.apache.sshd.client;version="[2.4.0,2.5.0)", - org.apache.sshd.client.auth;version="[2.4.0,2.5.0)", - org.apache.sshd.client.auth.keyboard;version="[2.4.0,2.5.0)", - org.apache.sshd.client.auth.password;version="[2.4.0,2.5.0)", - org.apache.sshd.client.auth.pubkey;version="[2.4.0,2.5.0)", - org.apache.sshd.client.channel;version="[2.4.0,2.5.0)", - org.apache.sshd.client.config.hosts;version="[2.4.0,2.5.0)", - org.apache.sshd.client.config.keys;version="[2.4.0,2.5.0)", - org.apache.sshd.client.future;version="[2.4.0,2.5.0)", - org.apache.sshd.client.keyverifier;version="[2.4.0,2.5.0)", - org.apache.sshd.client.session;version="[2.4.0,2.5.0)", - org.apache.sshd.client.session.forward;version="[2.4.0,2.5.0)", - org.apache.sshd.client.subsystem.sftp;version="[2.4.0,2.5.0)", - org.apache.sshd.common;version="[2.4.0,2.5.0)", - org.apache.sshd.common.auth;version="[2.4.0,2.5.0)", - org.apache.sshd.common.channel;version="[2.4.0,2.5.0)", - org.apache.sshd.common.compression;version="[2.4.0,2.5.0)", - org.apache.sshd.common.config.keys;version="[2.4.0,2.5.0)", - org.apache.sshd.common.config.keys.loader;version="[2.4.0,2.5.0)", - org.apache.sshd.common.config.keys.loader.openssh.kdf;version="[2.4.0,2.5.0)", - org.apache.sshd.common.digest;version="[2.4.0,2.5.0)", - org.apache.sshd.common.forward;version="[2.4.0,2.5.0)", - org.apache.sshd.common.future;version="[2.4.0,2.5.0)", - org.apache.sshd.common.helpers;version="[2.4.0,2.5.0)", - org.apache.sshd.common.io;version="[2.4.0,2.5.0)", - org.apache.sshd.common.kex;version="[2.4.0,2.5.0)", - org.apache.sshd.common.keyprovider;version="[2.4.0,2.5.0)", - org.apache.sshd.common.mac;version="[2.4.0,2.5.0)", - org.apache.sshd.common.random;version="[2.4.0,2.5.0)", - org.apache.sshd.common.session;version="[2.4.0,2.5.0)", - org.apache.sshd.common.session.helpers;version="[2.4.0,2.5.0)", - org.apache.sshd.common.signature;version="[2.4.0,2.5.0)", - org.apache.sshd.common.subsystem.sftp;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.buffer;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.closeable;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.io;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.io.resource;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.logging;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.net;version="[2.4.0,2.5.0)", - org.apache.sshd.common.util.security;version="[2.4.0,2.5.0)", - org.apache.sshd.server.auth;version="[2.4.0,2.5.0)", + org.apache.sshd.agent;version="[2.6.0,2.7.0)", + org.apache.sshd.client;version="[2.6.0,2.7.0)", + org.apache.sshd.client.auth;version="[2.6.0,2.7.0)", + org.apache.sshd.client.auth.keyboard;version="[2.6.0,2.7.0)", + org.apache.sshd.client.auth.password;version="[2.6.0,2.7.0)", + org.apache.sshd.client.auth.pubkey;version="[2.6.0,2.7.0)", + org.apache.sshd.client.channel;version="[2.6.0,2.7.0)", + org.apache.sshd.client.config.hosts;version="[2.6.0,2.7.0)", + org.apache.sshd.client.config.keys;version="[2.6.0,2.7.0)", + org.apache.sshd.client.future;version="[2.6.0,2.7.0)", + org.apache.sshd.client.keyverifier;version="[2.6.0,2.7.0)", + org.apache.sshd.client.session;version="[2.6.0,2.7.0)", + org.apache.sshd.client.session.forward;version="[2.6.0,2.7.0)", + org.apache.sshd.common;version="[2.6.0,2.7.0)", + org.apache.sshd.common.auth;version="[2.6.0,2.7.0)", + org.apache.sshd.common.channel;version="[2.6.0,2.7.0)", + org.apache.sshd.common.compression;version="[2.6.0,2.7.0)", + org.apache.sshd.common.config.keys;version="[2.6.0,2.7.0)", + org.apache.sshd.common.config.keys.loader;version="[2.6.0,2.7.0)", + org.apache.sshd.common.config.keys.loader.openssh.kdf;version="[2.6.0,2.7.0)", + org.apache.sshd.common.digest;version="[2.6.0,2.7.0)", + org.apache.sshd.common.forward;version="[2.6.0,2.7.0)", + org.apache.sshd.common.future;version="[2.6.0,2.7.0)", + org.apache.sshd.common.helpers;version="[2.6.0,2.7.0)", + org.apache.sshd.common.io;version="[2.6.0,2.7.0)", + org.apache.sshd.common.kex;version="[2.6.0,2.7.0)", + org.apache.sshd.common.keyprovider;version="[2.6.0,2.7.0)", + org.apache.sshd.common.mac;version="[2.6.0,2.7.0)", + org.apache.sshd.common.random;version="[2.6.0,2.7.0)", + org.apache.sshd.common.session;version="[2.6.0,2.7.0)", + org.apache.sshd.common.session.helpers;version="[2.6.0,2.7.0)", + org.apache.sshd.common.signature;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.buffer;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.closeable;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.io;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.io.resource;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.logging;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.net;version="[2.6.0,2.7.0)", + org.apache.sshd.common.util.security;version="[2.6.0,2.7.0)", + org.apache.sshd.core;version="[2.6.0,2.7.0)", + org.apache.sshd.server.auth;version="[2.6.0,2.7.0)", + org.apache.sshd.sftp;version="[2.6.0,2.7.0)", + org.apache.sshd.sftp.client;version="[2.6.0,2.7.0)", + org.apache.sshd.sftp.common;version="[2.6.0,2.7.0)", org.eclipse.jgit.annotations;version="[6.0.0,6.1.0)", org.eclipse.jgit.errors;version="[6.0.0,6.1.0)", org.eclipse.jgit.fnmatch;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java index 0d6f3027f..66713ba63 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java @@ -10,6 +10,7 @@ package org.eclipse.jgit.internal.transport.sshd; import static java.text.MessageFormat.format; +import static org.apache.sshd.core.CoreModuleProperties.MAX_IDENTIFICATION_SIZE; import java.io.IOException; import java.io.StreamCorruptedException; @@ -29,19 +30,14 @@ import org.apache.sshd.client.ClientFactoryManager; import org.apache.sshd.client.config.hosts.HostConfigEntry; -import org.apache.sshd.client.future.AuthFuture; import org.apache.sshd.client.keyverifier.ServerKeyVerifier; import org.apache.sshd.client.session.ClientSessionImpl; -import org.apache.sshd.client.session.ClientUserAuthService; import org.apache.sshd.common.AttributeRepository; import org.apache.sshd.common.FactoryManager; import org.apache.sshd.common.PropertyResolver; -import org.apache.sshd.common.PropertyResolverUtils; -import org.apache.sshd.common.SshException; import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.common.io.IoSession; import org.apache.sshd.common.io.IoWriteFuture; -import org.apache.sshd.common.kex.KexState; import org.apache.sshd.common.util.Readable; import org.apache.sshd.common.util.buffer.Buffer; import org.eclipse.jgit.errors.InvalidPatternException; @@ -66,7 +62,8 @@ public class JGitClientSession extends ClientSessionImpl { * protocol version exchange. 64kb is what OpenSSH < 8.0 read; OpenSSH 8.0 * changed it to 8Mb, but that seems excessive for the purpose stated in RFC * 4253. The Apache MINA sshd default in - * {@link FactoryManager#DEFAULT_MAX_IDENTIFICATION_SIZE} is 16kb. + * {@link org.apache.sshd.core.CoreModuleProperties#MAX_IDENTIFICATION_SIZE} + * is 16kb. */ private static final int DEFAULT_MAX_IDENTIFICATION_SIZE = 64 * 1024; @@ -76,17 +73,6 @@ public class JGitClientSession extends ClientSessionImpl { private volatile StatefulProxyConnector proxyHandler; - /** - * Work-around for bug 565394 / SSHD-1050; remove when using sshd 2.6.0. - */ - private volatile AuthFuture authFuture; - - /** Records exceptions before there is an authFuture. */ - private List earlyErrors = new ArrayList<>(); - - /** Guards setting an earlyError and the authFuture together. */ - private final Object errorLock = new Object(); - /** * @param manager * @param session @@ -97,125 +83,6 @@ public JGitClientSession(ClientFactoryManager manager, IoSession session) super(manager, session); } - // BEGIN Work-around for bug 565394 / SSHD-1050 - // Remove when using sshd 2.6.0. - - @Override - public AuthFuture auth() throws IOException { - if (getUsername() == null) { - throw new IllegalStateException( - SshdText.get().sessionWithoutUsername); - } - ClientUserAuthService authService = getUserAuthService(); - String serviceName = nextServiceName(); - List errors = null; - AuthFuture future; - // Guard both getting early errors and setting authFuture - synchronized (errorLock) { - future = authService.auth(serviceName); - if (future == null) { - // Internal error; no translation. - throw new IllegalStateException( - "No auth future generated by service '" //$NON-NLS-1$ - + serviceName + '\''); - } - errors = earlyErrors; - earlyErrors = null; - authFuture = future; - } - if (errors != null && !errors.isEmpty()) { - Iterator iter = errors.iterator(); - Throwable first = iter.next(); - iter.forEachRemaining(t -> { - if (t != first && t != null) { - first.addSuppressed(t); - } - }); - // Mark the future as having had an exception; just to be on the - // safe side. Actually, there shouldn't be anyone waiting on this - // future yet. - future.setException(first); - if (log.isDebugEnabled()) { - log.debug("auth({}) early exception type={}: {}", //$NON-NLS-1$ - this, first.getClass().getSimpleName(), - first.getMessage()); - } - if (first instanceof SshException) { - throw new SshException( - ((SshException) first).getDisconnectCode(), - first.getMessage(), first); - } - throw new IOException(first.getMessage(), first); - } - return future; - } - - @Override - protected void signalAuthFailure(AuthFuture future, Throwable t) { - signalAuthFailure(t); - } - - private void signalAuthFailure(Throwable t) { - AuthFuture future = authFuture; - if (future == null) { - synchronized (errorLock) { - if (earlyErrors != null) { - earlyErrors.add(t); - } - future = authFuture; - } - } - if (future != null) { - future.setException(t); - } - if (log.isDebugEnabled()) { - boolean signalled = future != null && t == future.getException(); - log.debug("signalAuthFailure({}) type={}, signalled={}: {}", this, //$NON-NLS-1$ - t.getClass().getSimpleName(), Boolean.valueOf(signalled), - t.getMessage()); - } - } - - @Override - public void exceptionCaught(Throwable t) { - signalAuthFailure(t); - super.exceptionCaught(t); - } - - @Override - protected void preClose() { - signalAuthFailure( - new SshException(SshdText.get().authenticationOnClosedSession)); - super.preClose(); - } - - @Override - protected void handleDisconnect(int code, String msg, String lang, - Buffer buffer) throws Exception { - signalAuthFailure(new SshException(code, msg)); - super.handleDisconnect(code, msg, lang, buffer); - } - - @Override - protected > C updateCurrentSessionState( - C newState) { - if (closeFuture.isClosed()) { - newState.add(ClientSessionEvent.CLOSED); - } - if (isAuthenticated()) { // authFuture.isSuccess() - newState.add(ClientSessionEvent.AUTHED); - } - if (KexState.DONE.equals(getKexState())) { - AuthFuture future = authFuture; - if (future == null || future.isFailure()) { - newState.add(ClientSessionEvent.WAIT_AUTH); - } - } - return newState; - } - - // END Work-around for bug 565394 / SSHD-1050 - /** * Retrieves the {@link HostConfigEntry} this session was created for. * @@ -331,22 +198,6 @@ public void messageReceived(Readable buffer) throws Exception { } } - @Override - protected void checkKeys() throws SshException { - ServerKeyVerifier serverKeyVerifier = getServerKeyVerifier(); - // The super implementation always uses - // getIoSession().getRemoteAddress(). In case of a proxy connection, - // that would be the address of the proxy! - SocketAddress remoteAddress = getConnectAddress(); - PublicKey serverKey = getKex().getServerKey(); - if (!serverKeyVerifier.verifyServerKey(this, remoteAddress, - serverKey)) { - throw new SshException( - org.apache.sshd.common.SshConstants.SSH2_DISCONNECT_HOST_KEY_NOT_VERIFIABLE, - SshdText.get().kexServerKeyInvalid); - } - } - @Override protected String resolveAvailableSignaturesProposal( FactoryManager manager) { @@ -477,9 +328,15 @@ protected List doReadIdentification(Buffer buffer, boolean server) throw new IllegalStateException( "doReadIdentification of client called with server=true"); //$NON-NLS-1$ } - int maxIdentSize = PropertyResolverUtils.getIntProperty(this, - FactoryManager.MAX_IDENTIFICATION_SIZE, - DEFAULT_MAX_IDENTIFICATION_SIZE); + Integer maxIdentLength = MAX_IDENTIFICATION_SIZE.get(this).orElse(null); + int maxIdentSize; + if (maxIdentLength == null || maxIdentLength + .intValue() < DEFAULT_MAX_IDENTIFICATION_SIZE) { + maxIdentSize = DEFAULT_MAX_IDENTIFICATION_SIZE; + MAX_IDENTIFICATION_SIZE.set(this, Integer.valueOf(maxIdentSize)); + } else { + maxIdentSize = maxIdentLength.intValue(); + } int current = buffer.rpos(); int end = current + buffer.available(); if (current >= end) { diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitPasswordAuthentication.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitPasswordAuthentication.java index 4abd6e901..ff8caaacc 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitPasswordAuthentication.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitPasswordAuthentication.java @@ -9,7 +9,8 @@ */ package org.eclipse.jgit.internal.transport.sshd; -import org.apache.sshd.client.ClientAuthenticationManager; +import static org.apache.sshd.core.CoreModuleProperties.PASSWORD_PROMPTS; + import org.apache.sshd.client.auth.keyboard.UserInteraction; import org.apache.sshd.client.auth.password.UserAuthPassword; import org.apache.sshd.client.session.ClientSession; @@ -29,9 +30,7 @@ public class JGitPasswordAuthentication extends UserAuthPassword { public void init(ClientSession session, String service) throws Exception { super.init(session, service); maxAttempts = Math.max(1, - session.getIntProperty( - ClientAuthenticationManager.PASSWORD_PROMPTS, - ClientAuthenticationManager.DEFAULT_PASSWORD_PROMPTS)); + PASSWORD_PROMPTS.getRequired(session).intValue()); attempts = 0; } diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java index beaaecaac..74455dc80 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshClient.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.transport.sshd; import static java.text.MessageFormat.format; +import static org.apache.sshd.core.CoreModuleProperties.PASSWORD_PROMPTS; +import static org.apache.sshd.core.CoreModuleProperties.PREFERRED_AUTHS; import static org.eclipse.jgit.internal.transport.ssh.OpenSshConfigFile.positive; import java.io.IOException; @@ -32,7 +34,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import org.apache.sshd.client.ClientAuthenticationManager; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.config.hosts.HostConfigEntry; import org.apache.sshd.client.future.ConnectFuture; @@ -169,12 +170,15 @@ private AttributeRepository sessionAttributes(AttributeRepository parent, Map, Object> data = new HashMap<>(); data.put(HOST_CONFIG_ENTRY, hostConfig); data.put(ORIGINAL_REMOTE_ADDRESS, originalAddress); + data.put(TARGET_SERVER, new SshdSocketAddress(originalAddress)); String preferredAuths = hostConfig.getProperty( SshConstants.PREFERRED_AUTHENTICATIONS, resolveAttribute(PREFERRED_AUTHENTICATIONS)); if (!StringUtils.isEmptyOrNull(preferredAuths)) { data.put(SessionAttributes.PROPERTIES, - Collections.singletonMap(PREFERRED_AUTHS, preferredAuths)); + Collections.singletonMap( + PREFERRED_AUTHS.getName(), + preferredAuths)); } return new SessionAttributes( AttributeRepository.ofAttributesMap(data), @@ -267,8 +271,7 @@ private JGitClientSession createSession(IoSession ioSession, session.setCredentialsProvider(getCredentialsProvider()); } int numberOfPasswordPrompts = getNumberOfPasswordPrompts(hostConfig); - session.getProperties().put(PASSWORD_PROMPTS, - Integer.valueOf(numberOfPasswordPrompts)); + PASSWORD_PROMPTS.set(session, Integer.valueOf(numberOfPasswordPrompts)); List identities = hostConfig.getIdentities().stream() .map(s -> { try { @@ -311,7 +314,7 @@ private int getNumberOfPasswordPrompts(HostConfigEntry hostConfig) { log.warn(format(SshdText.get().configInvalidPositive, SshConstants.NUMBER_OF_PASSWORD_PROMPTS, prompts)); } - return ClientAuthenticationManager.DEFAULT_PASSWORD_PROMPTS; + return PASSWORD_PROMPTS.getRequiredDefault().intValue(); } /** diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshConfig.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshConfig.java index 97e0fcc7d..6b0d9fb70 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshConfig.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitSshConfig.java @@ -46,7 +46,7 @@ public JGitSshConfig(SshConfigStore store) { @Override public HostConfigEntry resolveEffectiveHost(String host, int port, - SocketAddress localAddress, String username, + SocketAddress localAddress, String username, String proxyJump, AttributeRepository attributes) throws IOException { SshConfigStore.HostConfig entry = configFile == null ? SshConfigStore.EMPTY_CONFIG diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java index 5bc1115f6..47e09b75d 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/OpenSshServerKeyDatabase.java @@ -346,11 +346,14 @@ private void updateModifiedServerKey(PublicKey serverKey, throws IOException { KnownHostEntry hostEntry = entry.getHostEntry(); String oldLine = hostEntry.getConfigLine(); + if (oldLine == null) { + return; + } String newLine = updateHostKeyLine(oldLine, serverKey); if (newLine == null || newLine.isEmpty()) { return; } - if (oldLine == null || oldLine.isEmpty() || newLine.equals(oldLine)) { + if (oldLine.isEmpty() || newLine.equals(oldLine)) { // Shouldn't happen. return; } diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/PasswordProviderWrapper.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/PasswordProviderWrapper.java index 078e411f2..2cd066984 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/PasswordProviderWrapper.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/PasswordProviderWrapper.java @@ -9,6 +9,8 @@ */ package org.eclipse.jgit.internal.transport.sshd; +import static org.apache.sshd.core.CoreModuleProperties.PASSWORD_PROMPTS; + import java.io.IOException; import java.net.URISyntaxException; import java.security.GeneralSecurityException; @@ -18,7 +20,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import org.apache.sshd.client.ClientAuthenticationManager; import org.apache.sshd.common.AttributeRepository.AttributeKey; import org.apache.sshd.common.NamedResource; import org.apache.sshd.common.config.keys.FilePasswordProvider; @@ -62,15 +63,8 @@ private PerSessionState getState(SessionContext context) { if (state == null) { state = new PerSessionState(); state.delegate = factory.get(); - Integer maxNumberOfAttempts = context - .getInteger(ClientAuthenticationManager.PASSWORD_PROMPTS); - if (maxNumberOfAttempts != null - && maxNumberOfAttempts.intValue() > 0) { - state.delegate.setAttempts(maxNumberOfAttempts.intValue()); - } else { - state.delegate.setAttempts( - ClientAuthenticationManager.DEFAULT_PASSWORD_PROMPTS); - } + state.delegate.setAttempts( + PASSWORD_PROMPTS.getRequiredDefault().intValue()); context.setAttribute(STATE, state); } return state; diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/HttpClientConnector.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/HttpClientConnector.java index 8ac752bcc..e5d1e80f7 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/HttpClientConnector.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/HttpClientConnector.java @@ -135,7 +135,7 @@ private void send(StringBuilder msg, IoSession session) throws Exception { byte[] data = eol(msg).toString().getBytes(US_ASCII); Buffer buffer = new ByteArrayBuffer(data.length, false); buffer.putRawBytes(data); - session.writePacket(buffer).verify(getTimeout()); + session.writeBuffer(buffer).verify(getTimeout()); } private StringBuilder connect() { diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/Socks5ClientConnector.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/Socks5ClientConnector.java index 78b8d456b..8844efa6b 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/Socks5ClientConnector.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/Socks5ClientConnector.java @@ -235,7 +235,7 @@ public void sendClientProxyMetadata(ClientSession sshSession) buffer.putByte((byte) authenticationProposals.length); buffer.putRawBytes(authenticationProposals); state = ProtocolState.INIT; - session.writePacket(buffer).verify(getTimeout()); + session.writeBuffer(buffer).verify(getTimeout()); } private byte[] getAuthenticationProposals() { @@ -298,7 +298,7 @@ private void sendConnectInfo(IoSession session) throws Exception { buffer.putByte((byte) ((port >> 8) & 0xFF)); buffer.putByte((byte) (port & 0xFF)); state = ProtocolState.CONNECTING; - session.writePacket(buffer).verify(getTimeout()); + session.writeBuffer(buffer).verify(getTimeout()); } private void doPasswordAuth(IoSession session) throws Exception { @@ -335,7 +335,7 @@ private void startAuth(IoSession session) throws Exception { "No data for proxy authentication with " //$NON-NLS-1$ + proxyAddress); } - session.writePacket(buffer).verify(getTimeout()); + session.writeBuffer(buffer).verify(getTimeout()); } finally { if (buffer != null) { buffer.clear(true); @@ -350,7 +350,7 @@ private void authStep(IoSession session, Buffer input) throws Exception { authenticator.process(); buffer = authenticator.getToken(); if (buffer != null) { - session.writePacket(buffer).verify(getTimeout()); + session.writeBuffer(buffer).verify(getTimeout()); } } finally { if (buffer != null) { diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java index 0fb0610b9..33b234b1f 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSession.java @@ -11,6 +11,7 @@ import static java.text.MessageFormat.format; import static org.apache.sshd.common.SshConstants.SSH2_DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE; +import static org.apache.sshd.sftp.SftpModuleProperties.SFTP_CHANNEL_OPEN_TIMEOUT; import java.io.Closeable; import java.io.IOException; @@ -24,6 +25,7 @@ import java.util.EnumSet; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -37,23 +39,23 @@ import org.apache.sshd.client.future.ConnectFuture; import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.client.session.forward.PortForwardingTracker; -import org.apache.sshd.client.subsystem.sftp.SftpClient; -import org.apache.sshd.client.subsystem.sftp.SftpClient.CloseableHandle; -import org.apache.sshd.client.subsystem.sftp.SftpClient.CopyMode; -import org.apache.sshd.client.subsystem.sftp.SftpClientFactory; import org.apache.sshd.common.AttributeRepository; import org.apache.sshd.common.SshException; import org.apache.sshd.common.future.CloseFuture; import org.apache.sshd.common.future.SshFutureListener; -import org.apache.sshd.common.subsystem.sftp.SftpException; import org.apache.sshd.common.util.io.IoUtils; import org.apache.sshd.common.util.net.SshdSocketAddress; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.client.SftpClient.CloseableHandle; +import org.apache.sshd.sftp.client.SftpClient.CopyMode; +import org.apache.sshd.sftp.client.SftpClientFactory; +import org.apache.sshd.sftp.common.SftpException; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.transport.sshd.JGitSshClient; import org.eclipse.jgit.internal.transport.sshd.SshdText; import org.eclipse.jgit.transport.FtpChannel; -import org.eclipse.jgit.transport.RemoteSession; +import org.eclipse.jgit.transport.RemoteSession2; import org.eclipse.jgit.transport.SshConstants; import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.util.StringUtils; @@ -61,11 +63,12 @@ import org.slf4j.LoggerFactory; /** - * An implementation of {@link RemoteSession} based on Apache MINA sshd. + * An implementation of {@link org.eclipse.jgit.transport.RemoteSession + * RemoteSession} based on Apache MINA sshd. * * @since 5.2 */ -public class SshdSession implements RemoteSession { +public class SshdSession implements RemoteSession2 { private static final Logger LOG = LoggerFactory .getLogger(SshdSession.class); @@ -203,7 +206,7 @@ private void close(Closeable toClose, Throwable error) { private HostConfigEntry getHostConfig(String username, String host, int port) throws IOException { HostConfigEntry entry = client.getHostConfigEntryResolver() - .resolveEffectiveHost(host, port, null, username, null); + .resolveEffectiveHost(host, port, null, username, null, null); if (entry == null) { if (SshdSocketAddress.isIPv6Address(host)) { return new HostConfigEntry("", host, port, username); //$NON-NLS-1$ @@ -290,8 +293,15 @@ private void notifyCloseListeners() { @Override public Process exec(String commandName, int timeout) throws IOException { + return exec(commandName, Collections.emptyMap(), timeout); + } + + @Override + public Process exec(String commandName, Map environment, + int timeout) throws IOException { @SuppressWarnings("resource") - ChannelExec exec = session.createExecChannel(commandName); + ChannelExec exec = session.createExecChannel(commandName, null, + environment); if (timeout <= 0) { try { exec.open().verify(); @@ -430,13 +440,12 @@ private class SshdFtpChannel implements FtpChannel { @Override public void connect(int timeout, TimeUnit unit) throws IOException { if (timeout <= 0) { - session.getProperties().put( - SftpClient.SFTP_CHANNEL_OPEN_TIMEOUT, - Long.valueOf(Long.MAX_VALUE)); + // This timeout must not be null! + SFTP_CHANNEL_OPEN_TIMEOUT.set(session, + Duration.ofMillis(Long.MAX_VALUE)); } else { - session.getProperties().put( - SftpClient.SFTP_CHANNEL_OPEN_TIMEOUT, - Long.valueOf(unit.toMillis(timeout))); + SFTP_CHANNEL_OPEN_TIMEOUT.set(session, + Duration.ofMillis(unit.toMillis(timeout))); } ftp = SftpClientFactory.instance().createSftpClient(session); try { diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java index df0e1d28a..357994d43 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/transport/sshd/SshdSessionFactory.java @@ -35,10 +35,13 @@ import org.apache.sshd.client.auth.pubkey.UserAuthPublicKeyFactory; import org.apache.sshd.client.config.hosts.HostConfigEntryResolver; import org.apache.sshd.common.SshException; +import org.apache.sshd.common.NamedFactory; import org.apache.sshd.common.compression.BuiltinCompressions; import org.apache.sshd.common.config.keys.FilePasswordProvider; import org.apache.sshd.common.config.keys.loader.openssh.kdf.BCryptKdfOptions; import org.apache.sshd.common.keyprovider.KeyIdentityProvider; +import org.apache.sshd.common.signature.BuiltinSignatures; +import org.apache.sshd.common.signature.Signature; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.transport.ssh.OpenSshConfigFile; @@ -205,6 +208,7 @@ public SshdSession getSession(URIish uri, .hostConfigEntryResolver(configFile) .serverKeyVerifier(new JGitServerKeyVerifier( getServerKeyDatabase(home, sshDir))) + .signatureFactories(getSignatureFactories()) .compressionFactories( new ArrayList<>(BuiltinCompressions.VALUES)) .build(); @@ -590,4 +594,35 @@ private List getUserAuthFactories() { protected String getDefaultPreferredAuthentications() { return null; } + + /** + * Apache MINA sshd 2.6.0 has removed DSA, DSA_CERT and RSA_CERT. We have to + * set it up explicitly to still allow users to connect with DSA keys. + * + * @return a list of supported signature factories + */ + @SuppressWarnings("deprecation") + private static List> getSignatureFactories() { + // @formatter:off + return Arrays.asList( + BuiltinSignatures.nistp256_cert, + BuiltinSignatures.nistp384_cert, + BuiltinSignatures.nistp521_cert, + BuiltinSignatures.ed25519_cert, + BuiltinSignatures.rsaSHA512_cert, + BuiltinSignatures.rsaSHA256_cert, + BuiltinSignatures.rsa_cert, + BuiltinSignatures.nistp256, + BuiltinSignatures.nistp384, + BuiltinSignatures.nistp521, + BuiltinSignatures.ed25519, + BuiltinSignatures.sk_ecdsa_sha2_nistp256, + BuiltinSignatures.sk_ssh_ed25519, + BuiltinSignatures.rsaSHA512, + BuiltinSignatures.rsaSHA256, + BuiltinSignatures.rsa, + BuiltinSignatures.dsa_cert, + BuiltinSignatures.dsa); + // @formatter:on + } } diff --git a/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs index 2bc2cf30d..c16c98628 100644 --- a/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs @@ -52,8 +52,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.ssh.jsch.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.ssh.jsch.test/META-INF/MANIFEST.MF index 9fb299a1c..9ab1652a5 100644 --- a/org.eclipse.jgit.ssh.jsch.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.ssh.jsch.test/META-INF/MANIFEST.MF @@ -17,8 +17,4 @@ Import-Package: com.jcraft.jsch;version="[0.1.54,0.2.0)", org.junit;version="[4.13,5.0.0)", org.junit.experimental.theories;version="[4.13,5.0.0)", org.junit.runner;version="[4.13,5.0.0)" -Export-Package: org.eclipse.jgit.transport;version="6.0.0"; - uses:="org.eclipse.jgit.transport, - org.eclipse.jgit.junit, - org.eclipse.jgit.junit.ssh" Require-Bundle: org.hamcrest.core;bundle-version="[1.3.0,2.0.0)" diff --git a/org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java b/org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java new file mode 100644 index 000000000..0929c55c0 --- /dev/null +++ b/org.eclipse.jgit.ssh.jsch.test/tst/org/eclipse/jgit/transport/JSchSshProtocol2Test.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +//TODO(ms): move to org.eclipse.jgit.ssh.jsch in 6.0 +package org.eclipse.jgit.transport; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.util.Arrays; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.junit.ssh.SshBasicTestBase; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.transport.OpenSshConfig.Host; +import org.eclipse.jgit.util.FS; + +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; + +public class JSchSshProtocol2Test extends SshBasicTestBase { + + private class TestSshSessionFactory extends JschConfigSessionFactory { + + @Override + protected void configure(Host hc, Session session) { + // Nothing + } + + @Override + public synchronized RemoteSession getSession(URIish uri, + CredentialsProvider credentialsProvider, FS fs, int tms) + throws TransportException { + return super.getSession(uri, credentialsProvider, fs, tms); + } + + @Override + protected JSch createDefaultJSch(FS fs) throws JSchException { + JSch defaultJSch = super.createDefaultJSch(fs); + if (knownHosts.exists()) { + defaultJSch.setKnownHosts(knownHosts.getAbsolutePath()); + } + return defaultJSch; + } + } + + @Override + protected SshSessionFactory createSessionFactory() { + return new TestSshSessionFactory(); + } + + @Override + protected void installConfig(String... config) { + SshSessionFactory factory = getSessionFactory(); + assertTrue(factory instanceof JschConfigSessionFactory); + JschConfigSessionFactory j = (JschConfigSessionFactory) factory; + try { + j.setConfig(createConfig(config)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private OpenSshConfig createConfig(String... content) throws IOException { + File configFile = new File(sshDir, Constants.CONFIG); + if (content != null) { + Files.write(configFile.toPath(), Arrays.asList(content)); + } + return new OpenSshConfig(getTemporaryDirectory(), configFile); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + StoredConfig config = ((Repository) db).getConfig(); + config.setInt("protocol", null, "version", 2); + config.save(); + } +} diff --git a/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java b/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java index 858bdf3f7..c7d0941b6 100644 --- a/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java +++ b/org.eclipse.jgit.ssh.jsch/src/org/eclipse/jgit/transport/JschSession.java @@ -22,7 +22,9 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -44,7 +46,7 @@ * {@link org.eclipse.jgit.transport.JschConfigSessionFactory} is used to create * the actual session passed to the constructor. */ -public class JschSession implements RemoteSession { +public class JschSession implements RemoteSession2 { final Session sock; final URIish uri; @@ -65,7 +67,14 @@ public JschSession(Session session, URIish uri) { /** {@inheritDoc} */ @Override public Process exec(String command, int timeout) throws IOException { - return new JschProcess(command, timeout); + return exec(command, Collections.emptyMap(), timeout); + } + + /** {@inheritDoc} */ + @Override + public Process exec(String command, Map environment, + int timeout) throws IOException { + return new JschProcess(command, environment, timeout); } /** {@inheritDoc} */ @@ -124,6 +133,8 @@ private class JschProcess extends Process { * * @param commandName * the command to execute + * @param environment + * environment variables to pass on * @param tms * the timeout value, in seconds, for the command. * @throws TransportException @@ -132,11 +143,17 @@ private class JschProcess extends Process { * @throws IOException * on problems opening streams */ - JschProcess(String commandName, int tms) - throws TransportException, IOException { + JschProcess(String commandName, Map environment, + int tms) throws TransportException, IOException { timeout = tms; try { channel = (ChannelExec) sock.openChannel("exec"); //$NON-NLS-1$ + if (environment != null) { + for (Map.Entry envVar : environment + .entrySet()) { + channel.setEnv(envVar.getKey(), envVar.getValue()); + } + } channel.setCommand(commandName); setupStreams(); channel.connect(timeout > 0 ? timeout * 1000 : 0); diff --git a/org.eclipse.jgit.test/.settings/edu.umd.cs.findbugs.core.prefs b/org.eclipse.jgit.test/.settings/edu.umd.cs.findbugs.core.prefs new file mode 100644 index 000000000..70c173f8d --- /dev/null +++ b/org.eclipse.jgit.test/.settings/edu.umd.cs.findbugs.core.prefs @@ -0,0 +1,145 @@ +#SpotBugs User Preferences +#Fri Dec 04 11:35:48 CET 2020 +detectorExplicitSerialization=ExplicitSerialization|true +detectorMultithreadedInstanceAccess=MultithreadedInstanceAccess|true +detectorConfusionBetweenInheritedAndOuterMethod=ConfusionBetweenInheritedAndOuterMethod|true +detectorWrongMapIterator=WrongMapIterator|true +detectorUnnecessaryMath=UnnecessaryMath|true +detectorUselessSubclassMethod=UselessSubclassMethod|false +filter_settings=Medium|BAD_PRACTICE,CORRECTNESS,EXPERIMENTAL,I18N,MALICIOUS_CODE,MT_CORRECTNESS,PERFORMANCE,SECURITY,STYLE|false|15 +detectorURLProblems=URLProblems|true +detectorIteratorIdioms=IteratorIdioms|true +detectorMutableEnum=MutableEnum|true +detectorFindNonShortCircuit=FindNonShortCircuit|true +detectorSynchronizeAndNullCheckField=SynchronizeAndNullCheckField|true +detectorVolatileUsage=VolatileUsage|true +detectorFindNakedNotify=FindNakedNotify|true +detectorFindUninitializedGet=FindUninitializedGet|true +detectorFindUseOfNonSerializableValue=FindUseOfNonSerializableValue|true +detectorFindJSR166LockMonitorenter=FindJSR166LockMonitorenter|true +detectorQuestionableBooleanAssignment=QuestionableBooleanAssignment|true +detectorSwitchFallthrough=SwitchFallthrough|true +detectorFindLocalSelfAssignment2=FindLocalSelfAssignment2|true +detectorConfusedInheritance=ConfusedInheritance|true +detectorSynchronizationOnSharedBuiltinConstant=SynchronizationOnSharedBuiltinConstant|true +detectorMutableStaticFields=MutableStaticFields|true +detectorInvalidJUnitTest=InvalidJUnitTest|true +detectorInfiniteLoop=InfiniteLoop|true +detectorFindRunInvocations=FindRunInvocations|true +detectorBadSyntaxForRegularExpression=BadSyntaxForRegularExpression|true +detectorXMLFactoryBypass=XMLFactoryBypass|true +detectorFindOpenStream=FindOpenStream|true +detectorCheckExpectedWarnings=CheckExpectedWarnings|false +detectorHugeSharedStringConstants=HugeSharedStringConstants|true +detectorLostLoggerDueToWeakReference=LostLoggerDueToWeakReference|true +detectorStringConcatenation=StringConcatenation|true +detectorLoadOfKnownNullValue=LoadOfKnownNullValue|true +detectorFinalizerNullsFields=FinalizerNullsFields|true +detectorFindFieldSelfAssignment=FindFieldSelfAssignment|true +detectorInefficientToArray=InefficientToArray|false +detectorDontCatchIllegalMonitorStateException=DontCatchIllegalMonitorStateException|true +detectorInconsistentAnnotations=InconsistentAnnotations|true +detectorBadlyOverriddenAdapter=BadlyOverriddenAdapter|true +detectorInstantiateStaticClass=InstantiateStaticClass|true +detectorCheckRelaxingNullnessAnnotation=CheckRelaxingNullnessAnnotation|true +detectorMethodReturnCheck=MethodReturnCheck|true +detectorEqualsOperandShouldHaveClassCompatibleWithThis=EqualsOperandShouldHaveClassCompatibleWithThis|true +detectorFindDoubleCheck=FindDoubleCheck|true +detectorFindBadForLoop=FindBadForLoop|true +detectorDefaultEncodingDetector=DefaultEncodingDetector|true +detectorFindInconsistentSync2=FindInconsistentSync2|true +detectorFindSpinLoop=FindSpinLoop|true +detectorFindMaskedFields=FindMaskedFields|true +detectorBooleanReturnNull=BooleanReturnNull|true +detectorFindUnsyncGet=FindUnsyncGet|true +detectorCrossSiteScripting=CrossSiteScripting|true +detectorDroppedException=DroppedException|true +detectorFindDeadLocalStores=FindDeadLocalStores|true +detectorCheckImmutableAnnotation=CheckImmutableAnnotation|true +detectorInfiniteRecursiveLoop=InfiniteRecursiveLoop|true +detectorFindRefComparison=FindRefComparison|true +detectorFindRoughConstants=FindRoughConstants|true +detectorMutableLock=MutableLock|true +detectorFindNullDeref=FindNullDeref|true +detectorFindReturnRef=FindReturnRef|true +detectorSynchronizeOnClassLiteralNotGetClass=SynchronizeOnClassLiteralNotGetClass|true +detectorFindUselessControlFlow=FindUselessControlFlow|true +detectorOverridingEqualsNotSymmetrical=OverridingEqualsNotSymmetrical|true +detectorIDivResultCastToDouble=IDivResultCastToDouble|true +detectorReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass=ReadOfInstanceFieldInMethodInvokedByConstructorInSuperclass|true +detectorFindSelfComparison=FindSelfComparison|true +detectorFindFloatEquality=FindFloatEquality|true +detectorFindComparatorProblems=FindComparatorProblems|true +detectorRepeatedConditionals=RepeatedConditionals|true +filter_settings_neg=NOISE| +detectorInefficientMemberAccess=InefficientMemberAccess|false +detectorFindUncalledPrivateMethods=FindUncalledPrivateMethods|true +detectorNumberConstructor=NumberConstructor|true +detectorDontAssertInstanceofInTests=DontAssertInstanceofInTests|true +detectorFindFinalizeInvocations=FindFinalizeInvocations|true +detectorFindNullDerefsInvolvingNonShortCircuitEvaluation=FindNullDerefsInvolvingNonShortCircuitEvaluation|true +detectorDontIgnoreResultOfPutIfAbsent=DontIgnoreResultOfPutIfAbsent|true +detectorFindUnconditionalWait=FindUnconditionalWait|true +detectorFindTwoLockWait=FindTwoLockWait|true +detectorFindSleepWithLockHeld=FindSleepWithLockHeld|true +detectorFindUnreleasedLock=FindUnreleasedLock|true +detectorInefficientIndexOf=InefficientIndexOf|false +detectorDoInsideDoPrivileged=DoInsideDoPrivileged|true +detectorFindEmptySynchronizedBlock=FindEmptySynchronizedBlock|true +detectorOverridingMethodsMustInvokeSuperDetector=OverridingMethodsMustInvokeSuperDetector|true +detectorWaitInLoop=WaitInLoop|true +detectorIntCast2LongAsInstant=IntCast2LongAsInstant|true +detectorBadUseOfReturnValue=BadUseOfReturnValue|true +detectorFindSqlInjection=FindSqlInjection|true +detectorUnreadFields=UnreadFields|true +detectorSynchronizingOnContentsOfFieldToProtectField=SynchronizingOnContentsOfFieldToProtectField|true +detectorFindUselessObjects=FindUselessObjects|true +detectorBadAppletConstructor=BadAppletConstructor|false +detectorInheritanceUnsafeGetResource=InheritanceUnsafeGetResource|true +detectorSerializableIdiom=SerializableIdiom|true +detectorNaming=Naming|true +detectorNoteUnconditionalParamDerefs=NoteUnconditionalParamDerefs|true +detectorFormatStringChecker=FormatStringChecker|true +detectorSuspiciousThreadInterrupted=SuspiciousThreadInterrupted|true +detectorEmptyZipFileEntry=EmptyZipFileEntry|false +detectorFindCircularDependencies=FindCircularDependencies|false +detectorPreferZeroLengthArrays=PreferZeroLengthArrays|true +detectorAtomicityProblem=AtomicityProblem|true +detectorRuntimeExceptionCapture=RuntimeExceptionCapture|true +detectorInitializationChain=InitializationChain|true +detectorInitializeNonnullFieldsInConstructor=InitializeNonnullFieldsInConstructor|true +detectorOptionalReturnNull=OptionalReturnNull|true +detectorStartInConstructor=StartInConstructor|true +detectorFindUnsatisfiedObligation=FindUnsatisfiedObligation|true +detectorRedundantConditions=RedundantConditions|true +effort=default +detectorRedundantInterfaces=RedundantInterfaces|true +detectorDuplicateBranches=DuplicateBranches|true +detectorCheckTypeQualifiers=CheckTypeQualifiers|true +detectorComparatorIdiom=ComparatorIdiom|true +detectorFindBadCast2=FindBadCast2|true +detectorFindMismatchedWaitOrNotify=FindMismatchedWaitOrNotify|true +excludefilter0=findBugs/FindBugsExcludeFilter.xml|true +detectorBadResultSetAccess=BadResultSetAccess|true +detectorIncompatMask=IncompatMask|true +detectorCovariantArrayAssignment=CovariantArrayAssignment|false +detectorDumbMethodInvocations=DumbMethodInvocations|true +run_at_full_build=false +detectorStaticCalendarDetector=StaticCalendarDetector|true +detectorUncallableMethodOfAnonymousClass=UncallableMethodOfAnonymousClass|true +detectorVarArgsProblems=VarArgsProblems|true +detectorInefficientInitializationInsideLoop=InefficientInitializationInsideLoop|false +detectorCloneIdiom=CloneIdiom|true +detectorFindHEmismatch=FindHEmismatch|true +detectorAppendingToAnObjectOutputStream=AppendingToAnObjectOutputStream|true +detectorFindSelfComparison2=FindSelfComparison2|true +detectorLazyInit=LazyInit|true +detectorFindUnrelatedTypesInGenericContainer=FindUnrelatedTypesInGenericContainer|true +detectorDontUseEnum=DontUseEnum|true +detectorFindPuzzlers=FindPuzzlers|true +detectorCallToUnsupportedMethod=CallToUnsupportedMethod|false +detectorSuperfluousInstanceOf=SuperfluousInstanceOf|true +detectorReadReturnShouldBeChecked=ReadReturnShouldBeChecked|true +detector_threshold=2 +detectorPublicSemaphores=PublicSemaphores|false +detectorDumbMethods=DumbMethods|true diff --git a/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..b853c6a7e 100644 --- a/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs @@ -51,8 +51,8 @@ org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=error org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected -org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=no_tag +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.test/BUILD b/org.eclipse.jgit.test/BUILD index f12646e85..c9b5d3726 100644 --- a/org.eclipse.jgit.test/BUILD +++ b/org.eclipse.jgit.test/BUILD @@ -13,6 +13,8 @@ HELPERS = glob( ) + [PKG + c for c in [ "api/AbstractRemoteCommandTest.java", "diff/AbstractDiffTestCase.java", + "internal/revwalk/ObjectReachabilityTestCase.java", + "internal/revwalk/ReachabilityCheckerTestCase.java", "internal/storage/file/GcTestCase.java", "internal/storage/file/PackIndexTestCase.java", "internal/storage/file/XInputStream.java", @@ -20,8 +22,6 @@ HELPERS = glob( "nls/MissingPropertyBundle.java", "nls/NoPropertiesBundle.java", "nls/NonTranslatedBundle.java", - "revwalk/ObjectReachabilityTestCase.java", - "revwalk/ReachabilityCheckerTestCase.java", "revwalk/RevQueueTestCase.java", "revwalk/RevWalkTestCase.java", "transport/ObjectIdMatcher.java", @@ -44,8 +44,6 @@ EXCLUDED = [ PKG + "api/SecurityManagerMissingPermissionsTest.java", ] -RESOURCES = glob(["resources/**"]) - tests(tests = glob( ["tst/**/*.java"], exclude = HELPERS + DATA + EXCLUDED, diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index d4cc7b00b..ca6c58717 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -34,12 +34,12 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", org.eclipse.jgit.ignore.internal;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.fsck;version="[6.0.0,6.1.0)", + org.eclipse.jgit.internal.revwalk;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.dfs;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.file;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.io;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.pack;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.storage.reftable;version="[6.0.0,6.1.0)", - org.eclipse.jgit.internal.storage.reftree;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.transport.connectivity;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.transport.http;version="[6.0.0,6.1.0)", org.eclipse.jgit.internal.transport.parser;version="[6.0.0,6.1.0)", diff --git a/org.eclipse.jgit.test/findBugs/FindBugsExcludeFilter.xml b/org.eclipse.jgit.test/findBugs/FindBugsExcludeFilter.xml new file mode 100644 index 000000000..b4ef95379 --- /dev/null +++ b/org.eclipse.jgit.test/findBugs/FindBugsExcludeFilter.xml @@ -0,0 +1,9 @@ + + + + + + + diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java index 0f98a63f5..f2cceac4b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java @@ -12,6 +12,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.beans.Statement; import java.io.BufferedInputStream; @@ -28,6 +29,7 @@ import java.util.List; import java.util.Map; +import java.util.Random; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; @@ -55,6 +57,7 @@ import org.eclipse.jgit.util.StringUtils; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class ArchiveCommandTest extends RepositoryTestCase { @@ -184,9 +187,55 @@ public void archiveByDirectoryPath() throws GitAPIException, IOException { @Test public void archiveHeadAllFilesTarTimestamps() throws Exception { + archiveHeadAllFiles("tar"); + } + + @Test + public void archiveHeadAllFilesTgzTimestamps() throws Exception { + archiveHeadAllFiles("tgz"); + } + + @Test + public void archiveHeadAllFilesTbz2Timestamps() throws Exception { + archiveHeadAllFiles("tbz2"); + } + + @Test + public void archiveHeadAllFilesTxzTimestamps() throws Exception { + archiveHeadAllFiles("txz"); + } + + @Test + public void archiveHeadAllFilesZipTimestamps() throws Exception { + archiveHeadAllFiles("zip"); + } + + @Test + public void archiveHeadAllFilesTgzWithCompressionReducesArchiveSize() throws Exception { + archiveHeadAllFilesWithCompression("tgz"); + } + + @Test + public void archiveHeadAllFilesTbz2WithCompressionReducesArchiveSize() throws Exception { + archiveHeadAllFilesWithCompression("tbz2"); + } + + @Test + @Ignore + public void archiveHeadAllFilesTxzWithCompressionReducesArchiveSize() throws Exception { + // We ignore this test because the txz format consumes a lot of memory for high level + // compressions. + archiveHeadAllFilesWithCompression("txz"); + } + + @Test + public void archiveHeadAllFilesZipWithCompressionReducesArchiveSize() throws Exception { + archiveHeadAllFilesWithCompression("zip"); + } + + private void archiveHeadAllFiles(String fmt) throws Exception { try (Git git = new Git(db)) { createTestContent(git); - String fmt = "tar"; File archive = new File(getTemporaryDirectory(), "archive." + format); archive(git, archive, fmt); @@ -194,7 +243,7 @@ public void archiveHeadAllFilesTarTimestamps() throws Exception { try (InputStream fi = Files.newInputStream(archive.toPath()); InputStream bi = new BufferedInputStream(fi); - ArchiveInputStream o = new TarArchiveInputStream(bi)) { + ArchiveInputStream o = createArchiveInputStream(fmt, bi)) { assertEntries(o); } @@ -205,97 +254,42 @@ public void archiveHeadAllFilesTarTimestamps() throws Exception { } } - @Test - public void archiveHeadAllFilesTgzTimestamps() throws Exception { + @SuppressWarnings({ "serial", "boxing" }) + private void archiveHeadAllFilesWithCompression(String fmt) throws Exception { try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "tgz"; + createLargeTestContent(git); File archive = new File(getTemporaryDirectory(), - "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + "archive." + format); - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - InputStream gzi = new GzipCompressorInputStream(bi); - ArchiveInputStream o = new TarArchiveInputStream(gzi)) { - assertEntries(o); - } + archive(git, archive, fmt, new HashMap() {{ + put("compression-level", 1); + }}); + int sizeCompression1 = getNumBytes(archive); - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); + archive(git, archive, fmt, new HashMap() {{ + put("compression-level", 9); + }}); + int sizeCompression9 = getNumBytes(archive); + + assertTrue(sizeCompression1 > sizeCompression9); } } - @Test - public void archiveHeadAllFilesTbz2Timestamps() throws Exception { - try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "tbz2"; - File archive = new File(getTemporaryDirectory(), - "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); - - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - InputStream gzi = new BZip2CompressorInputStream(bi); - ArchiveInputStream o = new TarArchiveInputStream(gzi)) { - assertEntries(o); - } - - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); - } - } - - @Test - public void archiveHeadAllFilesTxzTimestamps() throws Exception { - try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "txz"; - File archive = new File(getTemporaryDirectory(), "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); - - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - InputStream gzi = new XZCompressorInputStream(bi); - ArchiveInputStream o = new TarArchiveInputStream(gzi)) { - assertEntries(o); - } - - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); - } - } - - @Test - public void archiveHeadAllFilesZipTimestamps() throws Exception { - try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "zip"; - File archive = new File(getTemporaryDirectory(), "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); - - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - ArchiveInputStream o = new ZipArchiveInputStream(bi)) { - assertEntries(o); - } - - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); + private static ArchiveInputStream createArchiveInputStream (String fmt, InputStream bi) + throws IOException { + switch (fmt) { + case "tar": + return new TarArchiveInputStream(bi); + case "tgz": + return new TarArchiveInputStream(new GzipCompressorInputStream(bi)); + case "tbz2": + return new TarArchiveInputStream(new BZip2CompressorInputStream(bi)); + case "txz": + return new TarArchiveInputStream(new XZCompressorInputStream(bi)); + case "zip": + return new ZipArchiveInputStream(new BufferedInputStream(bi)); } + throw new IllegalArgumentException("Format " + fmt + " is not supported."); } private void createTestContent(Git git) throws IOException, GitAPIException, @@ -312,13 +306,40 @@ private void createTestContent(Git git) throws IOException, GitAPIException, git.commit().setMessage("updated file").call(); } + private void createLargeTestContent(Git git) throws IOException, GitAPIException, + NoFilepatternException, NoHeadException, NoMessageException, + UnmergedPathsException, ConcurrentRefUpdateException, + WrongRepositoryStateException, AbortedByHookException { + StringBuilder largeContent = new StringBuilder(); + Random r = new Random(); + for (int i = 0; i < 2000; i++) { + for (int j = 0; j < 80; j++) { + largeContent.append((char)(r.nextInt(26) + 'a')); + } + largeContent.append("\n"); + } + writeTrashFile("large_file.txt", largeContent.toString()); + git.add().addFilepattern("large_file.txt").call(); + git.commit().setMessage("create file").call(); + } + private static void archive(Git git, File archive, String fmt) throws GitAPIException, FileNotFoundException, AmbiguousObjectException, IncorrectObjectTypeException, IOException { + archive(git, archive, fmt, new HashMap<>()); + } + + private static void archive(Git git, File archive, String fmt, Map options) + throws GitAPIException, + FileNotFoundException, AmbiguousObjectException, + IncorrectObjectTypeException, IOException { git.archive().setOutputStream(new FileOutputStream(archive)) .setFormat(fmt) - .setTree(git.getRepository().resolve("HEAD")).call(); + .setTree(git.getRepository().resolve("HEAD")) + .setFormatOptions(options) + .call(); } private static void assertEntries(ArchiveInputStream o) throws IOException { @@ -333,6 +354,13 @@ private static void assertEntries(ArchiveInputStream o) throws IOException { assertEquals(UNEXPECTED_ARCHIVE_SIZE, 2, n); } + private static int getNumBytes(File archive) throws Exception { + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi)) { + return bi.available(); + } + } + private static class MockFormat implements ArchiveCommand.Format { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java index 0a0a88c83..e52073251 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java @@ -146,6 +146,55 @@ public void testCheckoutForced_deleteFileAndRestore() throws Exception { assertTrue(testFile.exists()); } + @Test + public void testCheckoutForcedNoChangeNotInIndex() throws Exception { + git.checkout().setCreateBranch(true).setName("test2").call(); + File f = writeTrashFile("NewFile.txt", "New file"); + git.add().addFilepattern("NewFile.txt").call(); + git.commit().setMessage("New file created").call(); + git.checkout().setName("test").call(); + assertFalse("NewFile.txt should not exist", f.exists()); + writeTrashFile("NewFile.txt", "New file"); + git.add().addFilepattern("NewFile.txt").call(); + git.commit().setMessage("New file created again with same content") + .call(); + // Now remove the file from the index only. So it exists in both + // commits, and in the working tree, but not in the index. + git.rm().addFilepattern("NewFile.txt").setCached(true).call(); + assertTrue("NewFile.txt should exist", f.isFile()); + git.checkout().setForced(true).setName("test2").call(); + assertTrue("NewFile.txt should exist", f.isFile()); + assertEquals(Constants.R_HEADS + "test2", git.getRepository() + .exactRef(Constants.HEAD).getTarget().getName()); + assertTrue("Force checkout should have undone git rm --cached", + git.status().call().isClean()); + } + + @Test + public void testCheckoutNoChangeNotInIndex() throws Exception { + git.checkout().setCreateBranch(true).setName("test2").call(); + File f = writeTrashFile("NewFile.txt", "New file"); + git.add().addFilepattern("NewFile.txt").call(); + git.commit().setMessage("New file created").call(); + git.checkout().setName("test").call(); + assertFalse("NewFile.txt should not exist", f.exists()); + writeTrashFile("NewFile.txt", "New file"); + git.add().addFilepattern("NewFile.txt").call(); + git.commit().setMessage("New file created again with same content") + .call(); + // Now remove the file from the index only. So it exists in both + // commits, and in the working tree, but not in the index. + git.rm().addFilepattern("NewFile.txt").setCached(true).call(); + assertTrue("NewFile.txt should exist", f.isFile()); + git.checkout().setName("test2").call(); + assertTrue("NewFile.txt should exist", f.isFile()); + assertEquals(Constants.R_HEADS + "test2", git.getRepository() + .exactRef(Constants.HEAD).getTarget().getName()); + org.eclipse.jgit.api.Status status = git.status().call(); + assertEquals("[NewFile.txt]", status.getRemoved().toString()); + assertEquals("[NewFile.txt]", status.getUntracked().toString()); + } + @Test public void testCreateBranchOnCheckout() throws Exception { git.checkout().setCreateBranch(true).setName("test2").call(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/InitCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/InitCommandTest.java index 1c18b5a8b..48d835ed2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/InitCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/InitCommandTest.java @@ -9,6 +9,7 @@ */ package org.eclipse.jgit.api; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -21,8 +22,10 @@ import org.eclipse.jgit.errors.NoWorkTreeException; import org.eclipse.jgit.junit.MockSystemReader; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.util.SystemReader; import org.junit.Before; import org.junit.Test; @@ -42,7 +45,73 @@ public void testInitRepository() InitCommand command = new InitCommand(); command.setDirectory(directory); try (Git git = command.call()) { - assertNotNull(git.getRepository()); + Repository r = git.getRepository(); + assertNotNull(r); + assertEquals("refs/heads/master", r.getFullBranch()); + } + } + + @Test + public void testInitRepositoryMainInitialBranch() + throws IOException, JGitInternalException, GitAPIException { + File directory = createTempDirectory("testInitRepository"); + InitCommand command = new InitCommand(); + command.setDirectory(directory); + command.setInitialBranch("main"); + try (Git git = command.call()) { + Repository r = git.getRepository(); + assertNotNull(r); + assertEquals("refs/heads/main", r.getFullBranch()); + } + } + + @Test + public void testInitRepositoryCustomDefaultBranch() + throws Exception { + File directory = createTempDirectory("testInitRepository"); + InitCommand command = new InitCommand(); + command.setDirectory(directory); + MockSystemReader reader = (MockSystemReader) SystemReader.getInstance(); + StoredConfig c = reader.getUserConfig(); + String old = c.getString(ConfigConstants.CONFIG_INIT_SECTION, null, + ConfigConstants.CONFIG_KEY_DEFAULT_BRANCH); + c.setString(ConfigConstants.CONFIG_INIT_SECTION, null, + ConfigConstants.CONFIG_KEY_DEFAULT_BRANCH, "main"); + try (Git git = command.call()) { + Repository r = git.getRepository(); + assertNotNull(r); + assertEquals("refs/heads/main", r.getFullBranch()); + } finally { + c.setString(ConfigConstants.CONFIG_INIT_SECTION, null, + ConfigConstants.CONFIG_KEY_DEFAULT_BRANCH, old); + } + } + + @Test + public void testInitRepositoryNullInitialBranch() throws Exception { + File directory = createTempDirectory("testInitRepository"); + InitCommand command = new InitCommand(); + command.setDirectory(directory); + command.setInitialBranch("main"); + command.setInitialBranch(null); + try (Git git = command.call()) { + Repository r = git.getRepository(); + assertNotNull(r); + assertEquals("refs/heads/master", r.getFullBranch()); + } + } + + @Test + public void testInitRepositoryEmptyInitialBranch() throws Exception { + File directory = createTempDirectory("testInitRepository"); + InitCommand command = new InitCommand(); + command.setDirectory(directory); + command.setInitialBranch("main"); + command.setInitialBranch(""); + try (Git git = command.call()) { + Repository r = git.getRepository(); + assertNotNull(r); + assertEquals("refs/heads/master", r.getFullBranch()); } } @@ -72,6 +141,23 @@ public void testInitBareRepository() throws IOException, Repository repository = git.getRepository(); assertNotNull(repository); assertTrue(repository.isBare()); + assertEquals("refs/heads/master", repository.getFullBranch()); + } + } + + @Test + public void testInitBareRepositoryMainInitialBranch() + throws IOException, JGitInternalException, GitAPIException { + File directory = createTempDirectory("testInitBareRepository"); + InitCommand command = new InitCommand(); + command.setDirectory(directory); + command.setBare(true); + command.setInitialBranch("main"); + try (Git git = command.call()) { + Repository repository = git.getRepository(); + assertNotNull(repository); + assertTrue(repository.isBare()); + assertEquals("refs/heads/main", repository.getFullBranch()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LogCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LogCommandTest.java index 6460c7988..c563d5a47 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LogCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LogCommandTest.java @@ -232,6 +232,53 @@ public void logNoMergeCommits() throws Exception { assertFalse(i.hasNext()); } + /** + * 
    +	 * A - B - C - M
+	 *      \     /
+	 *        -D(side)
+	 *
    + */ + @Test + public void addRangeWithMerge() throws Exception{ + String fileA = "fileA"; + String fileB = "fileB"; + Git git = Git.wrap(db); + + writeTrashFile(fileA, fileA); + git.add().addFilepattern(fileA).call(); + git.commit().setMessage("commit a").call(); + + writeTrashFile(fileA, fileA); + git.add().addFilepattern(fileA).call(); + RevCommit b = git.commit().setMessage("commit b").call(); + + writeTrashFile(fileA, fileA); + git.add().addFilepattern(fileA).call(); + RevCommit c = git.commit().setMessage("commit c").call(); + + createBranch(b, "refs/heads/side"); + checkoutBranch("refs/heads/side"); + + writeTrashFile(fileB, fileB); + git.add().addFilepattern(fileB).call(); + RevCommit d = git.commit().setMessage("commit d").call(); + + checkoutBranch("refs/heads/master"); + MergeResult m = git.merge().include(d.getId()).call(); + assertEquals(MergeResult.MergeStatus.MERGED, m.getMergeStatus()); + + Iterator rangeLog = git.log().addRange(b.getId(), m.getNewHead()).call().iterator(); + + RevCommit commit = rangeLog.next(); + assertEquals(m.getNewHead(), commit.getId()); + commit = rangeLog.next(); + assertEquals(c.getId(), commit.getId()); + commit = rangeLog.next(); + assertEquals(d.getId(), commit.getId()); + assertFalse(rangeLog.hasNext()); + } + private void setCommitsAndMerge() throws Exception { Git git = Git.wrap(db); writeTrashFile("file1", "1\n2\n3\n4\n"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/TagCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/TagCommandTest.java index b1c54b9ef..99034174b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/TagCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/TagCommandTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013 Chris Aniszczyk and others + * Copyright (C) 2010, 2020 Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -11,6 +11,9 @@ import static org.eclipse.jgit.lib.Constants.R_TAGS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -19,8 +22,10 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidTagNameException; import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; @@ -28,6 +33,59 @@ public class TagCommandTest extends RepositoryTestCase { + @Test + public void testTagKind() { + try (Git git = new Git(db)) { + assertTrue(git.tag().isAnnotated()); + assertTrue(git.tag().setSigned(true).isAnnotated()); + assertTrue(git.tag().setSigned(false).isAnnotated()); + assertTrue(git.tag().setSigningKey(null).isAnnotated()); + assertTrue(git.tag().setSigningKey("something").isAnnotated()); + assertTrue(git.tag().setSigned(false).setSigningKey(null) + .isAnnotated()); + assertTrue(git.tag().setSigned(false).setSigningKey("something") + .isAnnotated()); + assertTrue(git.tag().setSigned(true).setSigningKey(null) + .isAnnotated()); + assertTrue(git.tag().setSigned(true).setSigningKey("something") + .isAnnotated()); + assertTrue(git.tag().setAnnotated(true).isAnnotated()); + assertTrue( + git.tag().setAnnotated(true).setSigned(true).isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigned(false) + .isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigningKey(null) + .isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigningKey("something") + .isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigned(false) + .setSigningKey(null).isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigned(false) + .setSigningKey("something").isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigned(true) + .setSigningKey(null).isAnnotated()); + assertTrue(git.tag().setAnnotated(true).setSigned(true) + .setSigningKey("something").isAnnotated()); + assertFalse(git.tag().setAnnotated(false).isAnnotated()); + assertTrue(git.tag().setAnnotated(false).setSigned(true) + .isAnnotated()); + assertFalse(git.tag().setAnnotated(false).setSigned(false) + .isAnnotated()); + assertFalse(git.tag().setAnnotated(false).setSigningKey(null) + .isAnnotated()); + assertTrue(git.tag().setAnnotated(false).setSigningKey("something") + .isAnnotated()); + assertFalse(git.tag().setAnnotated(false).setSigned(false) + .setSigningKey(null).isAnnotated()); + assertTrue(git.tag().setAnnotated(false).setSigned(false) + .setSigningKey("something").isAnnotated()); + assertTrue(git.tag().setAnnotated(false).setSigned(true) + .setSigningKey(null).isAnnotated()); + assertTrue(git.tag().setAnnotated(false).setSigned(true) + .setSigningKey("something").isAnnotated()); + } + } + @Test public void testTaggingOnHead() throws GitAPIException, IOException { try (Git git = new Git(db); @@ -66,6 +124,29 @@ public void testUnannotatedTagging() throws GitAPIException, } } + @Test + public void testForceNoChangeLightweight() throws GitAPIException { + try (Git git = new Git(db)) { + git.commit().setMessage("initial commit").call(); + RevCommit commit = git.commit().setMessage("second commit").call(); + git.commit().setMessage("third commit").call(); + Ref tagRef = git.tag().setObjectId(commit).setName("tag") + .setAnnotated(false).call(); + assertEquals(commit.getId(), tagRef.getObjectId()); + // Without force, we want to get a RefAlreadyExistsException + RefAlreadyExistsException e = assertThrows( + RefAlreadyExistsException.class, + () -> git.tag().setObjectId(commit).setName("tag") + .setAnnotated(false).call()); + assertEquals(RefUpdate.Result.NO_CHANGE, e.getUpdateResult()); + // With force the call should work + assertEquals(commit.getId(), + git.tag().setObjectId(commit).setName("tag") + .setAnnotated(false).setForceUpdate(true).call() + .getObjectId()); + } + } + @Test public void testEmptyTagName() throws GitAPIException { try (Git git = new Git(db)) { @@ -93,19 +174,6 @@ public void testInvalidTagName() throws GitAPIException { } } - @Test - public void testFailureOnSignedTags() throws GitAPIException { - try (Git git = new Git(db)) { - git.commit().setMessage("initial commit").call(); - try { - git.tag().setSigned(true).setName("tag").call(); - fail("We should have failed with an UnsupportedOperationException due to signed tag"); - } catch (UnsupportedOperationException e) { - // should hit here - } - } - } - private List getTags() throws Exception { return db.getRefDatabase().getRefsByPrefix(R_TAGS); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityTest.java similarity index 89% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityTest.java index d2b6e8916..f2f74050b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityTest.java @@ -7,11 +7,12 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.internal.storage.file.GC; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; public class BitmappedObjectReachabilityTest extends ObjectReachabilityTestCase { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedReachabilityCheckerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityCheckerTest.java similarity index 91% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedReachabilityCheckerTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityCheckerTest.java index 2cb88b979..5833c7a81 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedReachabilityCheckerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityCheckerTest.java @@ -7,13 +7,14 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import static org.junit.Assert.assertNotNull; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.internal.storage.file.GC; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.ReachabilityChecker; public class BitmappedReachabilityCheckerTest extends ReachabilityCheckerTestCase { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ObjectReachabilityTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java similarity index 95% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ObjectReachabilityTestCase.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java index 267b163f4..37ff40bdf 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ObjectReachabilityTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -20,6 +20,10 @@ import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.junit.TestRepository.CommitBuilder; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; import org.junit.Before; import org.junit.Test; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityTest.java similarity index 87% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityTest.java index b1c9556df..f06a76834 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityTest.java @@ -7,10 +7,11 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; public class PedestrianObjectReachabilityTest extends ObjectReachabilityTestCase { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianReachabilityCheckerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityCheckerTest.java similarity index 87% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianReachabilityCheckerTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityCheckerTest.java index 3029e056e..f9d4e182a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianReachabilityCheckerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityCheckerTest.java @@ -7,10 +7,11 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.ReachabilityChecker; public class PedestrianReachabilityCheckerTest extends ReachabilityCheckerTestCase { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ReachabilityCheckerTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java similarity index 96% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ReachabilityCheckerTestCase.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java index 4695eaa69..1ff6e7def 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ReachabilityCheckerTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -19,6 +19,8 @@ import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevCommit; import org.junit.Before; import org.junit.Test; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java index 18cf11717..b2c8ad5e7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java @@ -13,7 +13,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -82,7 +81,7 @@ public void objectLookupComparatorCustomPackSourceComparator() DfsPackDescription.objectLookupComparator( new PackSource.ComparatorBuilder() .add(GC) - .add(INSERT, RECEIVE, GC_REST, GC_TXN, UNREACHABLE_GARBAGE) + .add(INSERT, RECEIVE, GC_REST, UNREACHABLE_GARBAGE) .add(COMPACT) .build()), a, b); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java index 6fcd4ac05..dfd112976 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java @@ -14,7 +14,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.DEFAULT_COMPARATOR; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -30,7 +29,6 @@ public void defaultComaprator() throws Exception { assertEquals(0, DEFAULT_COMPARATOR.compare(COMPACT, COMPACT)); assertEquals(0, DEFAULT_COMPARATOR.compare(GC, GC)); assertEquals(0, DEFAULT_COMPARATOR.compare(GC_REST, GC_REST)); - assertEquals(0, DEFAULT_COMPARATOR.compare(GC_TXN, GC_TXN)); assertEquals(0, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, UNREACHABLE_GARBAGE)); assertEquals(0, DEFAULT_COMPARATOR.compare(INSERT, RECEIVE)); @@ -47,11 +45,5 @@ public void defaultComaprator() throws Exception { assertEquals(-1, DEFAULT_COMPARATOR.compare(GC, GC_REST)); assertEquals(1, DEFAULT_COMPARATOR.compare(GC_REST, GC)); - - assertEquals(-1, DEFAULT_COMPARATOR.compare(GC_REST, GC_TXN)); - assertEquals(1, DEFAULT_COMPARATOR.compare(GC_TXN, GC_REST)); - - assertEquals(-1, DEFAULT_COMPARATOR.compare(GC_TXN, UNREACHABLE_GARBAGE)); - assertEquals(1, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, GC_TXN)); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java index 72bff1683..6c74f0079 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableStackTest.java @@ -18,7 +18,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.Arrays; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -90,8 +91,9 @@ public void testCompaction(int N) throws Exception { assertEquals(ObjectId.zeroId(), c.getRef().getObjectId()); } - List files = Arrays.asList(reftableDir.listFiles()).stream() - .map(File::getName).collect(Collectors.toList()); + List files = Files.list(reftableDir.toPath()) + .map(Path::getFileName).map(Path::toString) + .collect(Collectors.toList()); Collections.sort(files); assertTrue(files.size() < 20); @@ -130,11 +132,14 @@ public void missingReftable() throws Exception { }); assertTrue(ok); - List files = Arrays.asList(reftableDir.listFiles()); + List files = Files.list(reftableDir.toPath()) + .collect(Collectors.toList()); for (int j = 0; j < files.size(); j++) { - File f = files.get(j); - if (f.getName().endsWith(".ref")) { - assertTrue(f.delete()); + Path f = files.get(j); + Path fileName = f.getFileName(); + if (fileName != null + && fileName.toString().endsWith(".ref")) { + Files.delete(f); break outer; } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java index 33bacbe3e..15c9109ca 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java @@ -28,14 +28,18 @@ import java.io.IOException; import java.security.SecureRandom; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefRename; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; @@ -579,6 +583,64 @@ public void reftableRefsStorageClass() throws IOException { assertEquals(Ref.Storage.PACKED, b.getStorage()); } + @Test + public void testGetRefsExcludingPrefix() throws IOException { + Set prefixes = new HashSet<>(); + prefixes.add("refs/tags"); + // HEAD + 12 refs/heads are present here. + List refs = + db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, prefixes); + assertEquals(13, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + checkContainsRef(refs, db.exactRef("refs/heads/a")); + for (Ref notInResult : db.getRefDatabase().getRefsByPrefix("refs/tags")) { + assertFalse(refs.contains(notInResult)); + } + } + + @Test + public void testGetRefsExcludingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/tags/"); + exclude.add("refs/heads/"); + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + } + + @Test + public void testGetRefsExcludingNonExistingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/tags/"); + exclude.add("refs/heads/"); + exclude.add("refs/nonexistent/"); + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + } + + @Test + public void testGetRefsWithPrefixExcludingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/heads/pa"); + String include = "refs/heads/p"; + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(include, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("refs/heads/prefix/a")); + } + + @Test + public void testGetRefsWithPrefixExcludingOverlappingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/heads/pa"); + exclude.add("refs/heads/"); + exclude.add("refs/heads/p"); + exclude.add("refs/tags/"); + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + } + private RefUpdate updateRef(String name) throws IOException { final RefUpdate ref = db.updateRef(name); ref.setNewObjectId(db.resolve(Constants.HEAD)); @@ -596,4 +658,14 @@ private void writeSymref(String src, String dst) throws IOException { fail("link " + src + " to " + dst); } } + + private static void checkContainsRef(Collection haystack, Ref needle) { + for (Ref ref : haystack) { + if (ref.getName().equals(needle.getName()) && + ref.getObjectId().equals(needle.getObjectId())) { + return; + } + } + fail("list " + haystack + " does not contain ref " + needle); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java index d007dd451..42e423845 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java @@ -157,7 +157,7 @@ public void testNotPackTwice(boolean aggressive) throws Exception { .create(); tr.update("refs/tags/t1", second); - Collection oldPacks = tr.getRepository().getObjectDatabase() + Collection oldPacks = tr.getRepository().getObjectDatabase() .getPacks(); assertEquals(0, oldPacks.size()); stats = gc.getStatistics(); @@ -171,7 +171,7 @@ public void testNotPackTwice(boolean aggressive) throws Exception { stats = gc.getStatistics(); assertEquals(0, stats.numberOfLooseObjects); - List packs = new ArrayList<>( + List packs = new ArrayList<>( repo.getObjectDatabase().getPacks()); assertEquals(11, packs.get(0).getObjectCount()); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java index bb8455f51..5cac1e342 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcConcurrentTest.java @@ -156,8 +156,8 @@ public void repackAndUploadPack() throws Exception { } } - PackFile getSinglePack(FileRepository r) { - Collection packs = r.getObjectDatabase().getPacks(); + Pack getSinglePack(FileRepository r) { + Collection packs = r.getObjectDatabase().getPacks(); assertEquals(1, packs.size()); return packs.iterator().next(); } @@ -206,11 +206,11 @@ public void testInterruptGc() throws Exception { SampleDataRepositoryTestCase.copyCGitTestPacks(repo); ExecutorService executor = Executors.newSingleThreadExecutor(); final CountDownLatch latch = new CountDownLatch(1); - Future> result = executor.submit(() -> { + Future> result = executor.submit(() -> { long start = System.currentTimeMillis(); System.out.println("starting gc"); latch.countDown(); - Collection r = gc.gc(); + Collection r = gc.gc(); System.out.println( "gc took " + (System.currentTimeMillis() - start) + " ms"); return r; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java index e1559584f..8472983d5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcKeepFilesTest.java @@ -36,9 +36,9 @@ public void testKeepFiles() throws Exception { assertEquals(4, stats.numberOfPackedObjects); assertEquals(1, stats.numberOfPackFiles); - Iterator packIt = repo.getObjectDatabase().getPacks() + Iterator packIt = repo.getObjectDatabase().getPacks() .iterator(); - PackFile singlePack = packIt.next(); + Pack singlePack = packIt.next(); assertFalse(packIt.hasNext()); String packFileName = singlePack.getPackFile().getPath(); String keepFileName = packFileName.substring(0, @@ -58,7 +58,7 @@ public void testKeepFiles() throws Exception { assertEquals(2, stats.numberOfPackFiles); // check that no object is packed twice - Iterator packs = repo.getObjectDatabase().getPacks() + Iterator packs = repo.getObjectDatabase().getPacks() .iterator(); PackIndex ind1 = packs.next().getIndex(); assertEquals(4, ind1.getObjectCount()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPruneNonReferencedTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPruneNonReferencedTest.java index 796df3db0..738662120 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPruneNonReferencedTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPruneNonReferencedTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.File; @@ -89,6 +90,7 @@ public void testPackCommitsAndLooseOneWithPruneNow() throws Exception { private void assertNoEmptyFanoutDirectories() { File[] fanout = repo.getObjectsDirectory().listFiles(); + assertNotNull(fanout); for (File f : fanout) { if (f.isDirectory()) { String[] entries = f.list(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java index ac65c3362..7c32ce7ce 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java @@ -11,6 +11,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; @@ -71,14 +72,14 @@ public void testSamePackDifferentCompressionDetectChecksumChanged() c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); c.save(); - Collection packs = gc(Deflater.NO_COMPRESSION); + Collection packs = gc(Deflater.NO_COMPRESSION); assertEquals("expected 1 packfile after gc", 1, packs.size()); - PackFile p1 = packs.iterator().next(); + Pack p1 = packs.iterator().next(); PackFileSnapshot snapshot = p1.getFileSnapshot(); packs = gc(Deflater.BEST_COMPRESSION); assertEquals("expected 1 packfile after gc", 1, packs.size()); - PackFile p2 = packs.iterator().next(); + Pack p2 = packs.iterator().next(); File pf = p2.getPackFile(); // changing compression level with aggressive gc may change size, @@ -152,11 +153,11 @@ public void testDetectModificationAlthoughSameSizeAndModificationtime() createTestRepo(testDataSeed, testDataLength); // repack to create initial packfile - PackFile pf = repackAndCheck(5, null, null, null); - Path packFilePath = pf.getPackFile().toPath(); - AnyObjectId chk1 = pf.getPackChecksum(); - String name = pf.getPackName(); - Long length = Long.valueOf(pf.getPackFile().length()); + Pack p = repackAndCheck(5, null, null, null); + Path packFilePath = p.getPackFile().toPath(); + AnyObjectId chk1 = p.getPackChecksum(); + String name = p.getPackName(); + Long length = Long.valueOf(p.getPackFile().length()); FS fs = db.getFS(); Instant m1 = fs.lastModifiedInstant(packFilePath); @@ -206,13 +207,16 @@ public void testDetectModificationAlthoughSameSizeAndModificationtimeAndFileKey( createTestRepo(testDataSeed, testDataLength); // Repack to create initial packfile. Make a copy of it - PackFile pf = repackAndCheck(5, null, null, null); - Path packFilePath = pf.getPackFile().toPath(); - Path packFileBasePath = packFilePath.resolveSibling( - packFilePath.getFileName().toString().replaceAll(".pack", "")); - AnyObjectId chk1 = pf.getPackChecksum(); - String name = pf.getPackName(); - Long length = Long.valueOf(pf.getPackFile().length()); + Pack p = repackAndCheck(5, null, null, null); + Path packFilePath = p.getPackFile().toPath(); + Path fn = packFilePath.getFileName(); + assertNotNull(fn); + String packFileName = fn.toString(); + Path packFileBasePath = packFilePath + .resolveSibling(packFileName.replaceAll(".pack", "")); + AnyObjectId chk1 = p.getPackChecksum(); + String name = p.getPackName(); + Long length = Long.valueOf(p.getPackFile().length()); copyPack(packFileBasePath, "", ".copy1"); // Repack to create second packfile. Make a copy of it @@ -276,10 +280,10 @@ private Path copyPack(Path base, String srcSuffix, String dstSuffix) Paths.get(base + ".pack" + dstSuffix)); } - private PackFile repackAndCheck(int compressionLevel, String oldName, + private Pack repackAndCheck(int compressionLevel, String oldName, Long oldLength, AnyObjectId oldChkSum) throws IOException, ParseException { - PackFile p = getSinglePack(gc(compressionLevel)); + Pack p = getSinglePack(gc(compressionLevel)); File pf = p.getPackFile(); // The following two assumptions should not cause the test to fail. If // on a certain platform we get packfiles (containing the same git @@ -294,14 +298,14 @@ private PackFile repackAndCheck(int compressionLevel, String oldName, return p; } - private PackFile getSinglePack(Collection packs) { - Iterator pIt = packs.iterator(); - PackFile p = pIt.next(); + private Pack getSinglePack(Collection packs) { + Iterator pIt = packs.iterator(); + Pack p = pIt.next(); assertFalse(pIt.hasNext()); return p; } - private Collection gc(int compressionLevel) + private Collection gc(int compressionLevel) throws IOException, ParseException { GC gc = new GC(db); PackConfig pc = new PackConfig(db.getConfig()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java index 8c56480fe..85043034a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java @@ -160,7 +160,7 @@ public void singlePack() throws Exception { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); assertEquals(3, packs.get(0).getObjectCount()); @@ -193,7 +193,7 @@ public void multiplePacks() throws Exception { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(2, packs.size()); assertEquals(1, packs.get(0).getObjectCount()); assertEquals(1, packs.get(1).getObjectCount()); @@ -216,9 +216,9 @@ public void largeBlob() throws Exception { } assertPacksOnly(); - Collection packs = listPacks(); + Collection packs = listPacks(); assertEquals(1, packs.size()); - PackFile p = packs.iterator().next(); + Pack p = packs.iterator().next(); assertEquals(1, p.getObjectCount()); try (ObjectReader reader = db.newObjectReader()) { @@ -237,9 +237,9 @@ public void overwriteExistingPack() throws Exception { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); - PackFile pack = packs.get(0); + Pack pack = packs.get(0); assertEquals(1, pack.getObjectCount()); String inode = getInode(pack.getPackFile()); @@ -372,7 +372,7 @@ public void readBackSmallFiles() throws Exception { } assertPacksOnly(); - List packs = listPacks(); + List packs = listPacks(); assertEquals(1, packs.size()); assertEquals(2, packs.get(0).getObjectCount()); @@ -489,16 +489,16 @@ public void readBackSmallObjectBeforeLargeObject() throws Exception { } } - private List listPacks() throws Exception { - List fromOpenDb = listPacks(db); - List reopened; + private List listPacks() throws Exception { + List fromOpenDb = listPacks(db); + List reopened; try (FileRepository db2 = new FileRepository(db.getDirectory())) { reopened = listPacks(db2); } assertEquals(fromOpenDb.size(), reopened.size()); for (int i = 0 ; i < fromOpenDb.size(); i++) { - PackFile a = fromOpenDb.get(i); - PackFile b = reopened.get(i); + Pack a = fromOpenDb.get(i); + Pack b = reopened.get(i); assertEquals(a.getPackName(), b.getPackName()); assertEquals( a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath()); @@ -508,9 +508,9 @@ private List listPacks() throws Exception { return fromOpenDb; } - private static List listPacks(FileRepository db) throws Exception { + private static List listPacks(FileRepository db) throws Exception { return db.getObjectDatabase().getPacks().stream() - .sorted(comparing(PackFile::getPackName)).collect(toList()); + .sorted(comparing(Pack::getPackName)).collect(toList()); } private PackInserter newInserter() { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java similarity index 94% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java index 97a86e249..182e42265 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackTest.java @@ -57,7 +57,7 @@ import org.junit.Before; import org.junit.Test; -public class PackFileTest extends LocalDiskRepositoryTestCase { +public class PackTest extends LocalDiskRepositoryTestCase { private int streamThreshold = 16 * 1024; private TestRng rng; @@ -228,21 +228,21 @@ public void testDelta_FailsOver2GiB() throws Exception { PackedObjectInfo a = new PackedObjectInfo(idA); PackedObjectInfo b = new PackedObjectInfo(idB); - TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024); - packHeader(pack, 2); - a.setOffset(pack.length()); - objectHeader(pack, Constants.OBJ_BLOB, base.length); - deflate(pack, base); + TemporaryBuffer.Heap packContents = new TemporaryBuffer.Heap(64 * 1024); + packHeader(packContents, 2); + a.setOffset(packContents.length()); + objectHeader(packContents, Constants.OBJ_BLOB, base.length); + deflate(packContents, base); ByteArrayOutputStream tmp = new ByteArrayOutputStream(); DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30); de.copy(0, 1); byte[] delta = tmp.toByteArray(); - b.setOffset(pack.length()); - objectHeader(pack, Constants.OBJ_REF_DELTA, delta.length); - idA.copyRawTo(pack); - deflate(pack, delta); - byte[] footer = digest(pack); + b.setOffset(packContents.length()); + objectHeader(packContents, Constants.OBJ_REF_DELTA, delta.length); + idA.copyRawTo(packContents); + deflate(packContents, delta); + byte[] footer = digest(packContents); File dir = new File(repo.getObjectDatabase().getDirectory(), "pack"); @@ -250,7 +250,7 @@ public void testDelta_FailsOver2GiB() throws Exception { File idxName = new File(dir, idA.name() + ".idx"); try (FileOutputStream f = new FileOutputStream(packName)) { - f.write(pack.toByteArray()); + f.write(packContents.toByteArray()); } try (FileOutputStream f = new FileOutputStream(idxName)) { @@ -261,14 +261,14 @@ public void testDelta_FailsOver2GiB() throws Exception { new PackIndexWriterV1(f).write(list, footer); } - PackFile packFile = new PackFile(packName, PackExt.INDEX.getBit()); + Pack pack = new Pack(packName, PackExt.INDEX.getBit()); try { - packFile.get(wc, b); + pack.get(wc, b); fail("expected LargeObjectException.ExceedsByteArrayLimit"); } catch (LargeObjectException.ExceedsByteArrayLimit bad) { assertNull(bad.getObjectId()); } finally { - packFile.close(); + pack.close(); } } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index c90310e07..214ddb989 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -72,7 +72,7 @@ public class PackWriterTest extends SampleDataRepositoryTestCase { private ByteArrayOutputStream os; - private PackFile pack; + private Pack pack; private ObjectInserter inserter; @@ -840,7 +840,7 @@ private void verifyOpenPack(boolean thin) throws IOException { p.setAllowThin(thin); p.setIndexVersion(2); p.parse(NullProgressMonitor.INSTANCE); - pack = p.getPackFile(); + pack = p.getPack(); assertNotNull("have PackFile after parsing", pack); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java index 97ef5993b..38c545ef5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java @@ -30,8 +30,10 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -352,6 +354,24 @@ public void testGetRefs_IgnoresGarbageRef4() throws IOException { assertEquals(A, c.getObjectId()); } + @Test + public void testGetRefs_ExcludingPrefixes() throws IOException { + writeLooseRef("refs/heads/A", A); + writeLooseRef("refs/heads/B", B); + writeLooseRef("refs/tags/tag", A); + writeLooseRef("refs/something/something", B); + writeLooseRef("refs/aaa/aaa", A); + + Set toExclude = new HashSet<>(); + toExclude.add("refs/aaa/"); + toExclude.add("refs/heads/"); + List refs = refdir.getRefsByPrefixWithExclusions(RefDatabase.ALL, toExclude); + + assertEquals(2, refs.size()); + assertTrue(refs.contains(refdir.exactRef("refs/tags/tag"))); + assertTrue(refs.contains(refdir.exactRef("refs/something/something"))); + } + @Test public void testFirstExactRef_IgnoresGarbageRef() throws IOException { writeLooseRef("refs/heads/A", A); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java index ee4c9b1dc..8f1371e09 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/T0004_PackReaderTest.java @@ -32,8 +32,8 @@ public void test003_lookupCompressedObject() throws IOException { final ObjectId id; final ObjectLoader or; - PackFile pr = null; - for (PackFile p : db.getObjectDatabase().getPacks()) { + Pack pr = null; + for (Pack p : db.getObjectDatabase().getPacks()) { if (PACK_NAME.equals(p.getPackName())) { pr = p; break; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java index 0a03fc352..9aea3b4b2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/MergedReftableTest.java @@ -137,6 +137,118 @@ public void twoTableSeek() throws IOException { } } + @Test + public void twoTableSeekPastWithRefCursor() throws IOException { + List delta1 = Arrays.asList( + ref("refs/heads/apple", 1), + ref("refs/heads/master", 2)); + List delta2 = Arrays.asList( + ref("refs/heads/banana", 3), + ref("refs/heads/zzlast", 4)); + + MergedReftable mr = merge(write(delta1), write(delta2)); + try (RefCursor rc = mr.seekRefsWithPrefix("")) { + assertTrue(rc.next()); + assertEquals("refs/heads/apple", rc.getRef().getName()); + assertEquals(id(1), rc.getRef().getObjectId()); + + rc.seekPastPrefix("refs/heads/banana/"); + + assertTrue(rc.next()); + assertEquals("refs/heads/master", rc.getRef().getName()); + assertEquals(id(2), rc.getRef().getObjectId()); + + assertTrue(rc.next()); + assertEquals("refs/heads/zzlast", rc.getRef().getName()); + assertEquals(id(4), rc.getRef().getObjectId()); + + assertEquals(1, rc.getRef().getUpdateIndex()); + } + } + + @Test + public void oneTableSeekPastWithRefCursor() throws IOException { + List delta1 = Arrays.asList( + ref("refs/heads/apple", 1), + ref("refs/heads/master", 2)); + + MergedReftable mr = merge(write(delta1)); + try (RefCursor rc = mr.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/apple"); + + assertTrue(rc.next()); + assertEquals("refs/heads/master", rc.getRef().getName()); + assertEquals(id(2), rc.getRef().getObjectId()); + + assertEquals(1, rc.getRef().getUpdateIndex()); + } + } + + @Test + public void seekPastToNonExistentPrefixToTheMiddle() throws IOException { + List delta1 = Arrays.asList( + ref("refs/heads/apple", 1), + ref("refs/heads/master", 2)); + List delta2 = Arrays.asList( + ref("refs/heads/banana", 3), + ref("refs/heads/zzlast", 4)); + + MergedReftable mr = merge(write(delta1), write(delta2)); + try (RefCursor rc = mr.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/x"); + + assertTrue(rc.next()); + assertEquals("refs/heads/zzlast", rc.getRef().getName()); + assertEquals(id(4), rc.getRef().getObjectId()); + + assertEquals(1, rc.getRef().getUpdateIndex()); + } + } + + @Test + public void seekPastToNonExistentPrefixToTheEnd() throws IOException { + List delta1 = Arrays.asList( + ref("refs/heads/apple", 1), + ref("refs/heads/master", 2)); + List delta2 = Arrays.asList( + ref("refs/heads/banana", 3), + ref("refs/heads/zzlast", 4)); + + MergedReftable mr = merge(write(delta1), write(delta2)); + try (RefCursor rc = mr.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/zzz"); + assertFalse(rc.next()); + } + } + + @Test + public void seekPastManyTimes() throws IOException { + List delta1 = Arrays.asList( + ref("refs/heads/apple", 1), + ref("refs/heads/master", 2)); + List delta2 = Arrays.asList( + ref("refs/heads/banana", 3), + ref("refs/heads/zzlast", 4)); + + MergedReftable mr = merge(write(delta1), write(delta2)); + try (RefCursor rc = mr.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/apple"); + rc.seekPastPrefix("refs/heads/banana"); + rc.seekPastPrefix("refs/heads/master"); + rc.seekPastPrefix("refs/heads/zzlast"); + assertFalse(rc.next()); + } + } + + @Test + public void seekPastOnEmptyTable() throws IOException { + MergedReftable mr = merge(write(), write()); + try (RefCursor rc = mr.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/"); + assertFalse(rc.next()); + } + } + @Test public void twoTableById() throws IOException { List delta1 = Arrays.asList( diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java index 009914b35..0d739b927 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftable/ReftableTest.java @@ -10,6 +10,7 @@ package org.eclipse.jgit.internal.storage.reftable; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; import static org.eclipse.jgit.lib.Constants.R_HEADS; @@ -49,8 +50,16 @@ import org.junit.Test; public class ReftableTest { + private static final byte[] LAST_UTF8_CHAR = new byte[] { + (byte)0x10, + (byte)0xFF, + (byte)0xFF}; + private static final String MASTER = "refs/heads/master"; private static final String NEXT = "refs/heads/next"; + private static final String AFTER_NEXT = "refs/heads/nextnext"; + private static final String LAST = "refs/heads/nextnextnext"; + private static final String NOT_REF_HEADS = "refs/zzz/zzz"; private static final String V1_0 = "refs/tags/v1.0"; private Stats stats; @@ -395,6 +404,135 @@ public void namespaceHeads() throws IOException { } } + @Test + public void seekPastRefWithRefCursor() throws IOException { + Ref exp = ref(MASTER, 1); + Ref next = ref(NEXT, 2); + Ref afterNext = ref(AFTER_NEXT, 3); + Ref afterNextNext = ref(LAST, 4); + ReftableReader t = read(write(exp, next, afterNext, afterNextNext)); + try (RefCursor rc = t.seekRefsWithPrefix("")) { + assertTrue(rc.next()); + assertEquals(MASTER, rc.getRef().getName()); + + rc.seekPastPrefix("refs/heads/next/"); + + assertTrue(rc.next()); + assertEquals(AFTER_NEXT, rc.getRef().getName()); + assertTrue(rc.next()); + assertEquals(LAST, rc.getRef().getName()); + + assertFalse(rc.next()); + } + } + + @Test + public void seekPastToNonExistentPrefixToTheMiddle() throws IOException { + Ref exp = ref(MASTER, 1); + Ref next = ref(NEXT, 2); + Ref afterNext = ref(AFTER_NEXT, 3); + Ref afterNextNext = ref(LAST, 4); + ReftableReader t = read(write(exp, next, afterNext, afterNextNext)); + try (RefCursor rc = t.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/master_non_existent"); + + assertTrue(rc.next()); + assertEquals(NEXT, rc.getRef().getName()); + + assertTrue(rc.next()); + assertEquals(AFTER_NEXT, rc.getRef().getName()); + + assertTrue(rc.next()); + assertEquals(LAST, rc.getRef().getName()); + + assertFalse(rc.next()); + } + } + + @Test + public void seekPastToNonExistentPrefixToTheEnd() throws IOException { + Ref exp = ref(MASTER, 1); + Ref next = ref(NEXT, 2); + Ref afterNext = ref(AFTER_NEXT, 3); + Ref afterNextNext = ref(LAST, 4); + ReftableReader t = read(write(exp, next, afterNext, afterNextNext)); + try (RefCursor rc = t.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/nextnon_existent_end"); + assertFalse(rc.next()); + } + } + + @Test + public void seekPastWithSeekRefsWithPrefix() throws IOException { + Ref exp = ref(MASTER, 1); + Ref next = ref(NEXT, 2); + Ref afterNext = ref(AFTER_NEXT, 3); + Ref afterNextNext = ref(LAST, 4); + Ref notRefsHeads = ref(NOT_REF_HEADS, 5); + ReftableReader t = read(write(exp, next, afterNext, afterNextNext, notRefsHeads)); + try (RefCursor rc = t.seekRefsWithPrefix("refs/heads/")) { + rc.seekPastPrefix("refs/heads/next/"); + assertTrue(rc.next()); + assertEquals(AFTER_NEXT, rc.getRef().getName()); + assertTrue(rc.next()); + assertEquals(LAST, rc.getRef().getName()); + + // NOT_REF_HEADS is next, but it's omitted because of + // seekRefsWithPrefix("refs/heads/"). + assertFalse(rc.next()); + } + } + + @Test + public void seekPastWithLotsOfRefs() throws IOException { + Ref[] refs = new Ref[500]; + for (int i = 1; i <= 500; i++) { + refs[i - 1] = ref(String.format("refs/%d", Integer.valueOf(i)), i); + } + ReftableReader t = read(write(refs)); + try (RefCursor rc = t.allRefs()) { + rc.seekPastPrefix("refs/3"); + assertTrue(rc.next()); + assertEquals("refs/4", rc.getRef().getName()); + assertTrue(rc.next()); + assertEquals("refs/40", rc.getRef().getName()); + + rc.seekPastPrefix("refs/8"); + assertTrue(rc.next()); + assertEquals("refs/9", rc.getRef().getName()); + assertTrue(rc.next()); + assertEquals("refs/90", rc.getRef().getName()); + assertTrue(rc.next()); + assertEquals("refs/91", rc.getRef().getName()); + } + } + + @Test + public void seekPastManyTimes() throws IOException { + Ref exp = ref(MASTER, 1); + Ref next = ref(NEXT, 2); + Ref afterNext = ref(AFTER_NEXT, 3); + Ref afterNextNext = ref(LAST, 4); + ReftableReader t = read(write(exp, next, afterNext, afterNextNext)); + + try (RefCursor rc = t.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/heads/master"); + rc.seekPastPrefix("refs/heads/next"); + rc.seekPastPrefix("refs/heads/nextnext"); + rc.seekPastPrefix("refs/heads/nextnextnext"); + assertFalse(rc.next()); + } + } + + @Test + public void seekPastOnEmptyTable() throws IOException { + ReftableReader t = read(write()); + try (RefCursor rc = t.seekRefsWithPrefix("")) { + rc.seekPastPrefix("refs/"); + assertFalse(rc.next()); + } + } + @Test public void indexScan() throws IOException { List refs = new ArrayList<>(); @@ -873,6 +1011,14 @@ public void byObjectIdOneRefWithIndex() throws IOException { } } + @Test + public void byObjectIdSkipPastPrefix() throws IOException { + ReftableReader t = read(write()); + try (RefCursor rc = t.byObjectId(id(2))) { + assertThrows(UnsupportedOperationException.class, () -> rc.seekPastPrefix("refs/heads/")); + } + } + @Test public void unpeeledDoesNotWrite() { try { @@ -883,6 +1029,18 @@ public void unpeeledDoesNotWrite() { } } + @Test + public void skipPastRefWithLastUTF8() throws IOException { + ReftableReader t = read(write(ref(String.format("refs/heads/%sbla", new String(LAST_UTF8_CHAR + , UTF_8)), 1))); + + try (RefCursor rc = t.allRefs()) { + rc.seekPastPrefix("refs/heads/"); + assertFalse(rc.next()); + } + } + + @Test public void nameTooLongDoesNotWrite() throws IOException { try { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/LocalDiskRefTreeDatabaseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/LocalDiskRefTreeDatabaseTest.java deleted file mode 100644 index 86016d8f7..000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/LocalDiskRefTreeDatabaseTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2016 Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.MASTER; -import static org.eclipse.jgit.lib.Constants.ORIG_HEAD; -import static org.eclipse.jgit.lib.Constants.R_HEADS; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.eclipse.jgit.internal.storage.file.FileRepository; -import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.junit.Before; -import org.junit.Test; - -public class LocalDiskRefTreeDatabaseTest extends LocalDiskRepositoryTestCase { - private FileRepository repo; - private RefTreeDatabase refdb; - private RefDatabase bootstrap; - - private TestRepository testRepo; - private RevCommit A; - private RevCommit B; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - FileRepository init = createWorkRepository(); - FileBasedConfig cfg = init.getConfig(); - cfg.setInt("core", null, "repositoryformatversion", 1); - cfg.setString("extensions", null, "refStorage", "reftree"); - cfg.save(); - - repo = (FileRepository) new FileRepositoryBuilder() - .setGitDir(init.getDirectory()) - .build(); - refdb = (RefTreeDatabase) repo.getRefDatabase(); - bootstrap = refdb.getBootstrap(); - addRepoToClose(repo); - - RefUpdate head = refdb.newUpdate(HEAD, true); - head.link(R_HEADS + MASTER); - - testRepo = new TestRepository<>(init); - A = testRepo.commit().create(); - B = testRepo.commit(testRepo.getRevWalk().parseCommit(A)); - } - - @Test - public void testHeadOrigHead() throws IOException { - RefUpdate master = refdb.newUpdate(HEAD, false); - master.setExpectedOldObjectId(ObjectId.zeroId()); - master.setNewObjectId(A); - assertEquals(RefUpdate.Result.NEW, master.update()); - assertEquals(A, refdb.exactRef(HEAD).getObjectId()); - - RefUpdate orig = refdb.newUpdate(ORIG_HEAD, true); - orig.setNewObjectId(B); - assertEquals(RefUpdate.Result.NEW, orig.update()); - - File origFile = new File(repo.getDirectory(), ORIG_HEAD); - assertEquals(B.name() + '\n', read(origFile)); - assertEquals(B, bootstrap.exactRef(ORIG_HEAD).getObjectId()); - assertEquals(B, refdb.exactRef(ORIG_HEAD).getObjectId()); - assertFalse(refdb.getRefs(ALL).containsKey(ORIG_HEAD)); - - List addl = refdb.getAdditionalRefs(); - assertEquals(2, addl.size()); - assertEquals(ORIG_HEAD, addl.get(1).getName()); - assertEquals(B, addl.get(1).getObjectId()); - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java deleted file mode 100644 index ecee5e5d2..000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java +++ /dev/null @@ -1,689 +0,0 @@ -/* - * Copyright (C) 2010, 2013, 2016 Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.ORIG_HEAD; -import static org.eclipse.jgit.lib.Constants.R_HEADS; -import static org.eclipse.jgit.lib.Constants.R_TAGS; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; -import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.junit.Before; -import org.junit.Test; - -public class RefTreeDatabaseTest { - private InMemRefTreeRepo repo; - private RefTreeDatabase refdb; - private RefDatabase bootstrap; - - private TestRepository testRepo; - private RevCommit A; - private RevCommit B; - private RevTag v1_0; - - @Before - public void setUp() throws Exception { - repo = new InMemRefTreeRepo(new DfsRepositoryDescription("test")); - bootstrap = refdb.getBootstrap(); - - testRepo = new TestRepository<>(repo); - A = testRepo.commit().create(); - B = testRepo.commit(testRepo.getRevWalk().parseCommit(A)); - v1_0 = testRepo.tag("v1_0", B); - testRepo.getRevWalk().parseBody(v1_0); - } - - @Test - public void testSupportsAtomic() { - assertTrue(refdb.performsAtomicTransactions()); - } - - @Test - public void testGetRefs_EmptyDatabase() throws IOException { - assertTrue("no references", refdb.getRefs(ALL).isEmpty()); - assertTrue("no references", refdb.getRefs(R_HEADS).isEmpty()); - assertTrue("no references", refdb.getRefs(R_TAGS).isEmpty()); - assertTrue("no references", refdb.getAdditionalRefs().isEmpty()); - } - - @Test - public void testGetAdditionalRefs() throws IOException { - update("refs/heads/master", A); - - List addl = refdb.getAdditionalRefs(); - assertEquals(1, addl.size()); - assertEquals("refs/txn/committed", addl.get(0).getName()); - assertEquals(getTxnCommitted(), addl.get(0).getObjectId()); - } - - @Test - public void testGetRefs_HeadOnOneBranch() throws IOException { - symref(HEAD, "refs/heads/master"); - update("refs/heads/master", A); - - Map all = refdb.getRefs(ALL); - assertEquals(2, all.size()); - assertTrue("has HEAD", all.containsKey(HEAD)); - assertTrue("has master", all.containsKey("refs/heads/master")); - - Ref head = all.get(HEAD); - Ref master = all.get("refs/heads/master"); - - assertEquals(HEAD, head.getName()); - assertTrue(head.isSymbolic()); - assertSame(LOOSE, head.getStorage()); - assertSame("uses same ref as target", master, head.getTarget()); - - assertEquals("refs/heads/master", master.getName()); - assertFalse(master.isSymbolic()); - assertSame(PACKED, master.getStorage()); - assertEquals(A, master.getObjectId()); - } - - @Test - public void testGetRefs_DetachedHead() throws IOException { - update(HEAD, A); - - Map all = refdb.getRefs(ALL); - assertEquals(1, all.size()); - assertTrue("has HEAD", all.containsKey(HEAD)); - - Ref head = all.get(HEAD); - assertEquals(HEAD, head.getName()); - assertFalse(head.isSymbolic()); - assertSame(PACKED, head.getStorage()); - assertEquals(A, head.getObjectId()); - } - - @Test - public void testGetRefs_DeeplyNestedBranch() throws IOException { - String name = "refs/heads/a/b/c/d/e/f/g/h/i/j/k"; - update(name, A); - - Map all = refdb.getRefs(ALL); - assertEquals(1, all.size()); - - Ref r = all.get(name); - assertEquals(name, r.getName()); - assertFalse(r.isSymbolic()); - assertSame(PACKED, r.getStorage()); - assertEquals(A, r.getObjectId()); - } - - @Test - public void testGetRefs_HeadBranchNotBorn() throws IOException { - update("refs/heads/A", A); - update("refs/heads/B", B); - - Map all = refdb.getRefs(ALL); - assertEquals(2, all.size()); - assertFalse("no HEAD", all.containsKey(HEAD)); - - Ref a = all.get("refs/heads/A"); - Ref b = all.get("refs/heads/B"); - - assertEquals(A, a.getObjectId()); - assertEquals(B, b.getObjectId()); - - assertEquals("refs/heads/A", a.getName()); - assertEquals("refs/heads/B", b.getName()); - } - - @Test - public void testGetRefs_HeadsOnly() throws IOException { - update("refs/heads/A", A); - update("refs/heads/B", B); - update("refs/tags/v1.0", v1_0); - - Map heads = refdb.getRefs(R_HEADS); - assertEquals(2, heads.size()); - - Ref a = heads.get("A"); - Ref b = heads.get("B"); - - assertEquals("refs/heads/A", a.getName()); - assertEquals("refs/heads/B", b.getName()); - - assertEquals(A, a.getObjectId()); - assertEquals(B, b.getObjectId()); - } - - @Test - public void testGetRefs_TagsOnly() throws IOException { - update("refs/heads/A", A); - update("refs/heads/B", B); - update("refs/tags/v1.0", v1_0); - - Map tags = refdb.getRefs(R_TAGS); - assertEquals(1, tags.size()); - - Ref a = tags.get("v1.0"); - assertEquals("refs/tags/v1.0", a.getName()); - assertEquals(v1_0, a.getObjectId()); - assertTrue(a.isPeeled()); - assertEquals(v1_0.getObject(), a.getPeeledObjectId()); - } - - @Test - public void testGetRefs_HeadsSymref() throws IOException { - symref("refs/heads/other", "refs/heads/master"); - update("refs/heads/master", A); - - Map heads = refdb.getRefs(R_HEADS); - assertEquals(2, heads.size()); - - Ref master = heads.get("master"); - Ref other = heads.get("other"); - - assertEquals("refs/heads/master", master.getName()); - assertEquals(A, master.getObjectId()); - - assertEquals("refs/heads/other", other.getName()); - assertEquals(A, other.getObjectId()); - assertSame(master, other.getTarget()); - } - - @Test - public void testGetRefs_InvalidPrefixes() throws IOException { - update("refs/heads/A", A); - - assertTrue("empty refs/heads", refdb.getRefs("refs/heads").isEmpty()); - assertTrue("empty objects", refdb.getRefs("objects").isEmpty()); - assertTrue("empty objects/", refdb.getRefs("objects/").isEmpty()); - } - - @Test - public void testGetRefs_DiscoversNew() throws IOException { - update("refs/heads/master", A); - Map orig = refdb.getRefs(ALL); - - update("refs/heads/next", B); - Map next = refdb.getRefs(ALL); - - assertEquals(1, orig.size()); - assertEquals(2, next.size()); - - assertFalse(orig.containsKey("refs/heads/next")); - assertTrue(next.containsKey("refs/heads/next")); - - assertEquals(A, next.get("refs/heads/master").getObjectId()); - assertEquals(B, next.get("refs/heads/next").getObjectId()); - } - - @Test - public void testGetRefs_DiscoversModified() throws IOException { - symref(HEAD, "refs/heads/master"); - update("refs/heads/master", A); - - Map all = refdb.getRefs(ALL); - assertEquals(A, all.get(HEAD).getObjectId()); - - update("refs/heads/master", B); - all = refdb.getRefs(ALL); - assertEquals(B, all.get(HEAD).getObjectId()); - assertEquals(B, refdb.exactRef(HEAD).getObjectId()); - } - - @Test - public void testGetRefs_CycleInSymbolicRef() throws IOException { - symref("refs/1", "refs/2"); - symref("refs/2", "refs/3"); - symref("refs/3", "refs/4"); - symref("refs/4", "refs/5"); - symref("refs/5", "refs/end"); - update("refs/end", A); - - Map all = refdb.getRefs(ALL); - Ref r = all.get("refs/1"); - assertNotNull("has 1", r); - - assertEquals("refs/1", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/2", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/3", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/4", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/5", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/end", r.getName()); - assertEquals(A, r.getObjectId()); - assertFalse(r.isSymbolic()); - - symref("refs/5", "refs/6"); - symref("refs/6", "refs/end"); - all = refdb.getRefs(ALL); - assertNull("mising 1 due to cycle", all.get("refs/1")); - assertEquals(A, all.get("refs/2").getObjectId()); - assertEquals(A, all.get("refs/3").getObjectId()); - assertEquals(A, all.get("refs/4").getObjectId()); - assertEquals(A, all.get("refs/5").getObjectId()); - assertEquals(A, all.get("refs/6").getObjectId()); - assertEquals(A, all.get("refs/end").getObjectId()); - } - - @Test - public void testGetRef_NonExistingBranchConfig() throws IOException { - assertNull("find branch config", refdb.findRef("config")); - assertNull("find branch config", refdb.findRef("refs/heads/config")); - } - - @Test - public void testGetRef_FindBranchConfig() throws IOException { - update("refs/heads/config", A); - - for (String t : new String[] { "config", "refs/heads/config" }) { - Ref r = refdb.findRef(t); - assertNotNull("find branch config (" + t + ")", r); - assertEquals("for " + t, "refs/heads/config", r.getName()); - assertEquals("for " + t, A, r.getObjectId()); - } - } - - @Test - public void testFirstExactRef() throws IOException { - update("refs/heads/A", A); - update("refs/tags/v1.0", v1_0); - - Ref a = refdb.firstExactRef("refs/heads/A", "refs/tags/v1.0"); - Ref one = refdb.firstExactRef("refs/tags/v1.0", "refs/heads/A"); - - assertEquals("refs/heads/A", a.getName()); - assertEquals("refs/tags/v1.0", one.getName()); - - assertEquals(A, a.getObjectId()); - assertEquals(v1_0, one.getObjectId()); - } - - @Test - public void testExactRef_DiscoversModified() throws IOException { - symref(HEAD, "refs/heads/master"); - update("refs/heads/master", A); - assertEquals(A, refdb.exactRef(HEAD).getObjectId()); - - update("refs/heads/master", B); - assertEquals(B, refdb.exactRef(HEAD).getObjectId()); - } - - @Test - public void testIsNameConflicting() throws IOException { - update("refs/heads/a/b", A); - update("refs/heads/q", B); - - // new references cannot replace an existing container - assertTrue(refdb.isNameConflicting("refs")); - assertTrue(refdb.isNameConflicting("refs/heads")); - assertTrue(refdb.isNameConflicting("refs/heads/a")); - - // existing reference is not conflicting - assertFalse(refdb.isNameConflicting("refs/heads/a/b")); - - // new references are not conflicting - assertFalse(refdb.isNameConflicting("refs/heads/a/d")); - assertFalse(refdb.isNameConflicting("refs/heads/master")); - - // existing reference must not be used as a container - assertTrue(refdb.isNameConflicting("refs/heads/a/b/c")); - assertTrue(refdb.isNameConflicting("refs/heads/q/master")); - - // refs/txn/ names always conflict. - assertTrue(refdb.isNameConflicting(refdb.getTxnCommitted())); - assertTrue(refdb.isNameConflicting("refs/txn/foo")); - } - - @Test - public void testUpdate_RefusesRefsTxnNamespace() throws IOException { - ObjectId txnId = getTxnCommitted(); - - RefUpdate u = refdb.newUpdate("refs/txn/tmp", false); - u.setNewObjectId(B); - assertEquals(RefUpdate.Result.LOCK_FAILURE, u.update()); - assertEquals(txnId, getTxnCommitted()); - - ReceiveCommand cmd = command(null, B, "refs/txn/tmp"); - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addCommand(cmd); - try (RevWalk rw = new RevWalk(repo)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); - assertEquals(MessageFormat.format(JGitText.get().invalidRefName, - "refs/txn/tmp"), cmd.getMessage()); - assertEquals(txnId, getTxnCommitted()); - } - - @Test - public void testUpdate_RefusesDotLockInRefName() throws IOException { - ObjectId txnId = getTxnCommitted(); - - RefUpdate u = refdb.newUpdate("refs/heads/pu.lock", false); - u.setNewObjectId(B); - assertEquals(RefUpdate.Result.REJECTED, u.update()); - assertEquals(txnId, getTxnCommitted()); - - ReceiveCommand cmd = command(null, B, "refs/heads/pu.lock"); - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addCommand(cmd); - try (RevWalk rw = new RevWalk(repo)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); - assertEquals(JGitText.get().funnyRefname, cmd.getMessage()); - assertEquals(txnId, getTxnCommitted()); - } - - @Test - public void testUpdate_RefusesOrigHeadOnBare() throws IOException { - assertTrue(refdb.getRepository().isBare()); - ObjectId txnId = getTxnCommitted(); - - RefUpdate orig = refdb.newUpdate(ORIG_HEAD, true); - orig.setNewObjectId(B); - assertEquals(RefUpdate.Result.LOCK_FAILURE, orig.update()); - assertEquals(txnId, getTxnCommitted()); - - ReceiveCommand cmd = command(null, B, ORIG_HEAD); - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addCommand(cmd); - try (RevWalk rw = new RevWalk(repo)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); - assertEquals( - MessageFormat.format(JGitText.get().invalidRefName, ORIG_HEAD), - cmd.getMessage()); - assertEquals(txnId, getTxnCommitted()); - } - - @Test - public void testBatchRefUpdate_NonFastForwardAborts() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(B, A, "refs/heads/masters")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(txnId, getTxnCommitted()); - - assertEquals(REJECTED_NONFASTFORWARD, - commands.get(1).getResult()); - assertEquals(REJECTED_OTHER_REASON, - commands.get(0).getResult()); - assertEquals(JGitText.get().transactionAborted, - commands.get(0).getMessage()); - } - - @Test - public void testBatchRefUpdate_ForceUpdate() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(B, A, "refs/heads/masters")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertNotEquals(txnId, getTxnCommitted()); - - Map refs = refdb.getRefs(ALL); - assertEquals(OK, commands.get(0).getResult()); - assertEquals(OK, commands.get(1).getResult()); - assertEquals( - "[refs/heads/master, refs/heads/masters]", - refs.keySet().toString()); - assertEquals(B.getId(), refs.get("refs/heads/master").getObjectId()); - assertEquals(A.getId(), refs.get("refs/heads/masters").getObjectId()); - } - - @Test - public void testBatchRefUpdate_NonFastForwardDoesNotDoExpensiveMergeCheck() - throws IOException { - update("refs/heads/master", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(B, A, "refs/heads/master")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo) { - @Override - public boolean isMergedInto(RevCommit base, RevCommit tip) { - fail("isMergedInto() should not be called"); - return false; - } - }) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertNotEquals(txnId, getTxnCommitted()); - - Map refs = refdb.getRefs(ALL); - assertEquals(OK, commands.get(0).getResult()); - assertEquals(A.getId(), refs.get("refs/heads/master").getObjectId()); - } - - @Test - public void testBatchRefUpdate_ConflictCausesAbort() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(null, A, "refs/heads/master/x"), - command(null, A, "refs/heads")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(txnId, getTxnCommitted()); - - assertEquals(LOCK_FAILURE, commands.get(0).getResult()); - - assertEquals(REJECTED_OTHER_REASON, commands.get(1).getResult()); - assertEquals(JGitText.get().transactionAborted, - commands.get(1).getMessage()); - - assertEquals(REJECTED_OTHER_REASON, commands.get(2).getResult()); - assertEquals(JGitText.get().transactionAborted, - commands.get(2).getMessage()); - } - - @Test - public void testBatchRefUpdate_NoConflictIfDeleted() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(null, A, "refs/heads/masters/x"), - command(B, null, "refs/heads/masters")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertNotEquals(txnId, getTxnCommitted()); - - assertEquals(OK, commands.get(0).getResult()); - assertEquals(OK, commands.get(1).getResult()); - assertEquals(OK, commands.get(2).getResult()); - - Map refs = refdb.getRefs(ALL); - assertEquals( - "[refs/heads/master, refs/heads/masters/x]", - refs.keySet().toString()); - assertEquals(A.getId(), refs.get("refs/heads/masters/x").getObjectId()); - } - - private ObjectId getTxnCommitted() throws IOException { - Ref r = bootstrap.exactRef(refdb.getTxnCommitted()); - if (r != null && r.getObjectId() != null) { - return r.getObjectId(); - } - return ObjectId.zeroId(); - } - - private static ReceiveCommand command(AnyObjectId a, AnyObjectId b, - String name) { - return new ReceiveCommand( - a != null ? a.copy() : ObjectId.zeroId(), - b != null ? b.copy() : ObjectId.zeroId(), - name); - } - - private void symref(String name, String dst) - throws IOException { - commit((ObjectReader reader, RefTree tree) -> { - Ref old = tree.exactRef(reader, name); - Command n = new Command(old, new SymbolicRef(name, - new ObjectIdRef.Unpeeled(Ref.Storage.NEW, dst, null))); - return tree.apply(Collections.singleton(n)); - }); - } - - private void update(String name, ObjectId id) - throws IOException { - commit((ObjectReader reader, RefTree tree) -> { - Ref old = tree.exactRef(reader, name); - Command n; - try (RevWalk rw = new RevWalk(repo)) { - n = new Command(old, Command.toRef(rw, id, null, name, true)); - } - return tree.apply(Collections.singleton(n)); - }); - } - - interface Function { - boolean apply(ObjectReader reader, RefTree tree) throws IOException; - } - - private void commit(Function fun) throws IOException { - try (ObjectReader reader = repo.newObjectReader(); - ObjectInserter inserter = repo.newObjectInserter(); - RevWalk rw = new RevWalk(reader)) { - RefUpdate u = bootstrap.newUpdate(refdb.getTxnCommitted(), false); - CommitBuilder cb = new CommitBuilder(); - testRepo.setAuthorAndCommitter(cb); - - Ref ref = bootstrap.exactRef(refdb.getTxnCommitted()); - RefTree tree; - if (ref != null && ref.getObjectId() != null) { - tree = RefTree.read(reader, rw.parseTree(ref.getObjectId())); - cb.setParentId(ref.getObjectId()); - u.setExpectedOldObjectId(ref.getObjectId()); - } else { - tree = RefTree.newEmptyTree(); - u.setExpectedOldObjectId(ObjectId.zeroId()); - } - - assertTrue(fun.apply(reader, tree)); - cb.setTreeId(tree.writeTree(inserter)); - u.setNewObjectId(inserter.insert(cb)); - inserter.flush(); - switch (u.update(rw)) { - case NEW: - case FAST_FORWARD: - break; - default: - fail("Expected " + u.getName() + " to update"); - } - } - } - - private class InMemRefTreeRepo extends InMemoryRepository { - private final RefTreeDatabase refs; - - InMemRefTreeRepo(DfsRepositoryDescription repoDesc) { - super(repoDesc); - refs = new RefTreeDatabase(this, super.getRefDatabase(), - "refs/txn/committed"); - RefTreeDatabaseTest.this.refdb = refs; - } - - @Override - public RefDatabase getRefDatabase() { - return refs; - } - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeTest.java deleted file mode 100644 index a5b01900c..000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeTest.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.R_HEADS; -import static org.eclipse.jgit.lib.Constants.R_TAGS; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; - -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; -import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevBlob; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.junit.Before; -import org.junit.Test; - -public class RefTreeTest { - private static final String R_MASTER = R_HEADS + "master"; - private InMemoryRepository repo; - private TestRepository git; - - @Before - public void setUp() throws IOException { - repo = new InMemoryRepository(new DfsRepositoryDescription("RefTree")); - git = new TestRepository<>(repo); - } - - @Test - public void testEmptyTree() throws IOException { - RefTree tree = RefTree.newEmptyTree(); - try (ObjectReader reader = repo.newObjectReader()) { - assertNull(HEAD, tree.exactRef(reader, HEAD)); - assertNull("master", tree.exactRef(reader, R_MASTER)); - } - } - - @Test - public void testApplyThenReadMaster() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob id = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, id)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - assertSame(NOT_ATTEMPTED, cmd.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, R_MASTER); - assertNotNull(R_MASTER, m); - assertEquals(R_MASTER, m.getName()); - assertEquals(id, m.getObjectId()); - assertTrue("peeled", m.isPeeled()); - } - } - - @Test - public void testUpdateMaster() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob id1 = git.blob("A"); - Command cmd1 = new Command(null, ref(R_MASTER, id1)); - assertTrue(tree.apply(Collections.singletonList(cmd1))); - assertSame(NOT_ATTEMPTED, cmd1.getResult()); - - RevBlob id2 = git.blob("B"); - Command cmd2 = new Command(ref(R_MASTER, id1), ref(R_MASTER, id2)); - assertTrue(tree.apply(Collections.singletonList(cmd2))); - assertSame(NOT_ATTEMPTED, cmd2.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, R_MASTER); - assertNotNull(R_MASTER, m); - assertEquals(R_MASTER, m.getName()); - assertEquals(id2, m.getObjectId()); - assertTrue("peeled", m.isPeeled()); - } - } - - @Test - public void testHeadSymref() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob id = git.blob("A"); - Command cmd1 = new Command(null, ref(R_MASTER, id)); - Command cmd2 = new Command(null, symref(HEAD, R_MASTER)); - assertTrue(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); - assertSame(NOT_ATTEMPTED, cmd1.getResult()); - assertSame(NOT_ATTEMPTED, cmd2.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, HEAD); - assertNotNull(HEAD, m); - assertEquals(HEAD, m.getName()); - assertTrue("symbolic", m.isSymbolic()); - assertNotNull(m.getTarget()); - assertEquals(R_MASTER, m.getTarget().getName()); - assertEquals(id, m.getTarget().getObjectId()); - } - - // Writing flushes some buffers, re-read from blob. - ObjectId newId = write(tree); - try (ObjectReader reader = repo.newObjectReader(); - RevWalk rw = new RevWalk(reader)) { - tree = RefTree.read(reader, rw.parseTree(newId)); - Ref m = tree.exactRef(reader, HEAD); - assertEquals(R_MASTER, m.getTarget().getName()); - } - } - - @Test - public void testTagIsPeeled() throws Exception { - String name = "v1.0"; - RefTree tree = RefTree.newEmptyTree(); - RevBlob id = git.blob("A"); - RevTag tag = git.tag(name, id); - - String ref = R_TAGS + name; - Command cmd = create(ref, tag); - assertTrue(tree.apply(Collections.singletonList(cmd))); - assertSame(NOT_ATTEMPTED, cmd.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, ref); - assertNotNull(ref, m); - assertEquals(ref, m.getName()); - assertEquals(tag, m.getObjectId()); - assertTrue("peeled", m.isPeeled()); - assertEquals(id, m.getPeeledObjectId()); - } - } - - @Test - public void testApplyAlreadyExists() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - Command cmd1 = create(R_MASTER, b); - Command cmd2 = create(R_MASTER, b); - assertFalse(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertSame(REJECTED_OTHER_REASON, cmd2.getResult()); - assertEquals(JGitText.get().transactionAborted, cmd2.getMessage()); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyWrongOldId() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - RevBlob c = git.blob("C"); - Command cmd1 = update(R_MASTER, b, c); - Command cmd2 = create(R_MASTER, b); - assertFalse(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertSame(REJECTED_OTHER_REASON, cmd2.getResult()); - assertEquals(JGitText.get().transactionAborted, cmd2.getMessage()); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyWrongOldIdButAlreadyCurrentIsNoOp() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - cmd = update(R_MASTER, b, a); - assertTrue(tree.apply(Collections.singletonList(cmd))); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyCannotCreateSubdirectory() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - Command cmd1 = create(R_MASTER + "/fail", b); - assertFalse(tree.apply(Collections.singletonList(cmd1))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyCannotCreateParentRef() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - Command cmd1 = create("refs/heads", b); - assertFalse(tree.apply(Collections.singletonList(cmd1))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertEquals(treeId, write(tree)); - } - - private static Ref ref(String name, ObjectId id) { - return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); - } - - private static Ref symref(String name, String dest) { - Ref d = new ObjectIdRef.PeeledNonTag(NEW, dest, null); - return new SymbolicRef(name, d); - } - - private Command create(String name, ObjectId id) - throws MissingObjectException, IOException { - return update(name, ObjectId.zeroId(), id); - } - - private Command update(String name, ObjectId oldId, ObjectId newId) - throws MissingObjectException, IOException { - try (RevWalk rw = new RevWalk(repo)) { - return new Command(rw, new ReceiveCommand(oldId, newId, name)); - } - } - - private ObjectId write(RefTree tree) throws IOException { - try (ObjectInserter ins = repo.newObjectInserter()) { - ObjectId id = tree.writeTree(ins); - ins.flush(); - return id; - } - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/CommitBuilderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/CommitBuilderTest.java index dee58f9cf..2f1bada82 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/CommitBuilderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/CommitBuilderTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Salesforce. and others + * Copyright (C) 2018, 2020 Salesforce. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -53,7 +53,7 @@ public class CommitBuilderTest { private void assertGpgSignatureStringOutcome(String signature, String expectedOutcome) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - CommitBuilder.writeGpgSignatureString(signature, out); + ObjectBuilder.writeMultiLineHeader(signature, out, true); String formatted_signature = new String(out.toByteArray(), US_ASCII); assertEquals(expectedOutcome, formatted_signature); } @@ -85,8 +85,8 @@ public void writeGpgSignatureString_failsForNonAscii() throws Exception { String signature = "Ü Ä"; IllegalArgumentException e = assertThrows( IllegalArgumentException.class, - () -> CommitBuilder.writeGpgSignatureString(signature, - new ByteArrayOutputStream())); + () -> ObjectBuilder.writeMultiLineHeader(signature, + new ByteArrayOutputStream(), true)); String message = MessageFormat.format(JGitText.get().notASCIIString, signature); assertEquals(message, e.getMessage()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefTest.java index 88d17ec15..7590048a7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefTest.java @@ -26,6 +26,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -317,6 +318,64 @@ public void testGetRefsByPrefixes() throws IOException { checkContainsRef(refs, db.exactRef("refs/tags/A")); } + @Test + public void testGetRefsExcludingPrefix() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/tags"); + // HEAD + 12 refs/heads are present here. + List refs = + db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, exclude); + assertEquals(13, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + checkContainsRef(refs, db.exactRef("refs/heads/a")); + for (Ref notInResult : db.getRefDatabase().getRefsByPrefix("refs/tags")) { + assertFalse(refs.contains(notInResult)); + } + } + + @Test + public void testGetRefsExcludingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/tags/"); + exclude.add("refs/heads/"); + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + } + + @Test + public void testGetRefsExcludingNonExistingPrefixes() throws IOException { + Set prefixes = new HashSet<>(); + prefixes.add("refs/tags/"); + prefixes.add("refs/heads/"); + prefixes.add("refs/nonexistent/"); + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, prefixes); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + } + + @Test + public void testGetRefsWithPrefixExcludingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/heads/pa"); + String include = "refs/heads/p"; + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(include, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("refs/heads/prefix/a")); + } + + @Test + public void testGetRefsWithPrefixExcludingOverlappingPrefixes() throws IOException { + Set exclude = new HashSet<>(); + exclude.add("refs/heads/pa"); + exclude.add("refs/heads/"); + exclude.add("refs/heads/p"); + exclude.add("refs/tags/"); + List refs = db.getRefDatabase().getRefsByPrefixWithExclusions(RefDatabase.ALL, exclude); + assertEquals(1, refs.size()); + checkContainsRef(refs, db.exactRef("HEAD")); + } + @Test public void testResolveTipSha1() throws IOException { ObjectId masterId = db.resolve("refs/heads/master"); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TagBuilderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TagBuilderTest.java new file mode 100644 index 000000000..578602224 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/TagBuilderTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.util.RawParseUtils; +import org.junit.Test; + +public class TagBuilderTest { + + // @formatter:off + private static final String SIGNATURE = "-----BEGIN PGP SIGNATURE-----\n" + + "Version: BCPG v1.60\n" + + "\n" + + "iQEcBAABCAAGBQJb9cVhAAoJEKX+6Axg/6TZeFsH/0CY0WX/z7U8+7S5giFX4wH4\n" + + "opvBwqyt6OX8lgNwTwBGHFNt8LdmDCCmKoq/XwkNi3ARVjLhe3gBcKXNoavvPk2Z\n" + + "gIg5ChevGkU4afWCOMLVEYnkCBGw2+86XhrK1P7gTHEk1Rd+Yv1ZRDJBY+fFO7yz\n" + + "uSBuF5RpEY2sJiIvp27Gub/rY3B5NTR/feO/z+b9oiP/fMUhpRwG5KuWUsn9NPjw\n" + + "3tvbgawYpU/2UnS+xnavMY4t2fjRYjsoxndPLb2MUX8X7vC7FgWLBlmI/rquLZVM\n" + + "IQEKkjnA+lhejjK1rv+ulq4kGZJFKGYWYYhRDwFg5PTkzhudhN2SGUq5Wxq1Eg4=\n" + + "=b9OI\n" + + "-----END PGP SIGNATURE-----"; + + // @formatter:on + + private static final String TAGGER_LINE = "A U. Thor 1218123387 +0700"; + + private static final PersonIdent TAGGER = RawParseUtils + .parsePersonIdent(TAGGER_LINE); + + @Test + public void testTagSimple() throws Exception { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setEncoding(US_ASCII); + t.setMessage("Short message only"); + t.setTagger(TAGGER); + String tag = new String(t.build(), UTF_8); + String expected = "object 0000000000000000000000000000000000000000\n" + + "type commit\n" // + + "tag sometag\n" // + + "tagger " + TAGGER_LINE + '\n' // + + "encoding US-ASCII\n" // + + '\n' // + + "Short message only"; + assertEquals(expected, tag); + } + + @Test + public void testTagWithSignatureShortMessageEndsInLF() throws Exception { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setEncoding(US_ASCII); + t.setMessage("Short message only\n"); + t.setTagger(TAGGER); + t.setGpgSignature(new GpgSignature(SIGNATURE.getBytes(US_ASCII))); + String tag = new String(t.build(), UTF_8); + String expected = "object 0000000000000000000000000000000000000000\n" + + "type commit\n" // + + "tag sometag\n" // + + "tagger " + TAGGER_LINE + '\n' // + + "encoding US-ASCII\n" // + + '\n' // + + "Short message only\n" // + + SIGNATURE + '\n'; + assertEquals(expected, tag); + } + + @Test + public void testTagWithSignatureMessageNoLF() { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setEncoding(US_ASCII); + t.setMessage("A message\n\nthat does not end in LF"); + t.setTagger(TAGGER); + t.setGpgSignature(new GpgSignature(SIGNATURE.getBytes(US_ASCII))); + Throwable ex = assertThrows(Throwable.class, t::build); + assertEquals(JGitText.get().signedTagMessageNoLf, ex.getMessage()); + } + + @Test + public void testTagWithSignatureNoParagraphsMessage() throws Exception { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setEncoding(US_ASCII); + t.setMessage("A strange\ntag message\n"); + t.setTagger(TAGGER); + t.setGpgSignature(new GpgSignature(SIGNATURE.getBytes(US_ASCII))); + String tag = new String(t.build(), UTF_8); + String expected = "object 0000000000000000000000000000000000000000\n" + + "type commit\n" // + + "tag sometag\n" // + + "tagger " + TAGGER_LINE + '\n' // + + "encoding US-ASCII\n" // + + '\n' // + + "A strange\ntag message\n" // + + SIGNATURE + '\n'; + assertEquals(expected, tag); + } + + @Test + public void testTagWithSignatureLongMessage() throws Exception { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setMessage("Short message\n\nFollowed by explanations.\n"); + t.setTagger(TAGGER); + t.setGpgSignature(new GpgSignature(SIGNATURE.getBytes(US_ASCII))); + String tag = new String(t.build(), UTF_8); + String expected = "object 0000000000000000000000000000000000000000\n" + + "type commit\n" // + + "tag sometag\n" // + + "tagger " + TAGGER_LINE + '\n' // + + '\n' // + + "Short message\n\nFollowed by explanations.\n" // + + SIGNATURE + '\n'; + assertEquals(expected, tag); + } + + @Test + public void testTagWithSignatureEmptyMessage() throws Exception { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setTagger(TAGGER); + t.setMessage(""); + String emptyMsg = new String(t.build(), UTF_8); + t.setGpgSignature(new GpgSignature(SIGNATURE.getBytes(US_ASCII))); + String tag = new String(t.build(), UTF_8); + String expected = "object 0000000000000000000000000000000000000000\n" + + "type commit\n" // + + "tag sometag\n" // + + "tagger " + TAGGER_LINE + '\n' // + + '\n'; + assertEquals(expected, emptyMsg); + assertEquals(expected + SIGNATURE + '\n', tag); + } + + @Test + public void testTagWithSignatureOnly() throws Exception { + TagBuilder t = new TagBuilder(); + t.setTag("sometag"); + t.setObjectId(ObjectId.zeroId(), Constants.OBJ_COMMIT); + t.setTagger(TAGGER); + String emptyMsg = new String(t.build(), UTF_8); + t.setGpgSignature(new GpgSignature(SIGNATURE.getBytes(US_ASCII))); + String tag = new String(t.build(), UTF_8); + String expected = "object 0000000000000000000000000000000000000000\n" + + "type commit\n" // + + "tag sometag\n" // + + "tagger " + TAGGER_LINE + '\n' // + + '\n'; + assertEquals(expected, emptyMsg); + assertEquals(expected + SIGNATURE + '\n', tag); + } + +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java index e2ac89be9..eecf25be9 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java @@ -1384,6 +1384,270 @@ public void checkMergeConflictInVirtualAncestor( git.merge().include(commitB).call(); } + /** + * Merging two commits with a file/dir conflict in the virtual ancestor. + * + *

    + * Those conflicts should be ignored, otherwise the found base can not be used by the + * RecursiveMerger. + * 

    +	 *  --------------
+	 * |              \
+	 * |         C1 - C4 --- ?     master
+	 * |        /          /
+	 * |  I - A1 - C2 - C3         second-branch
+	 * |   \            /
+	 * \    \          /
+	 *  ----A2--------             branch-to-merge
+	 *
    + *

    + *

    + * Path "a" is initially a file in I and A1. It is changed to a directory in A2 + * ("branch-to-merge"). + *

    + * A2 is merged into "master" and "second-branch". The dir/file merge conflict is resolved + * manually, results in C4 and C3. + *

    + * While merging C3 and C4, A1 and A2 are the base commits found by the recursive merge that + * have the dir/file conflict. + */ + @Theory + public void checkFileDirMergeConflictInVirtualAncestor_NoConflictInChildren( + MergeStrategy strategy) + throws Exception { + if (!strategy.equals(MergeStrategy.RECURSIVE)) { + return; + } + + Git git = Git.wrap(db); + + // master + writeTrashFile("a", "initial content"); + git.add().addFilepattern("a").call(); + RevCommit commitI = git.commit().setMessage("Initial commit").call(); + + writeTrashFile("a", "content in Ancestor 1"); + git.add().addFilepattern("a").call(); + RevCommit commitA1 = git.commit().setMessage("Ancestor 1").call(); + + writeTrashFile("a", "content in Child 1 (commited on master)"); + git.add().addFilepattern("a").call(); + // commit C1M + git.commit().setMessage("Child 1 on master").call(); + + git.checkout().setCreateBranch(true).setStartPoint(commitI).setName("branch-to-merge").call(); + // "a" becomes a directory in A2 + git.rm().addFilepattern("a").call(); + writeTrashFile("a/content", "content in Ancestor 2 (commited on branch-to-merge)"); + git.add().addFilepattern("a/content").call(); + RevCommit commitA2 = git.commit().setMessage("Ancestor 2").call(); + + // second branch + git.checkout().setCreateBranch(true).setStartPoint(commitA1).setName("second-branch").call(); + writeTrashFile("a", "content in Child 2 (commited on second-branch)"); + git.add().addFilepattern("a").call(); + // commit C2S + git.commit().setMessage("Child 2 on second-branch").call(); + + // Merge branch-to-merge into second-branch + MergeResult mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + // Resolve the conflict manually, merge "a" as a file + git.rm().addFilepattern("a").call(); + git.rm().addFilepattern("a/content").call(); + writeTrashFile("a", "merge conflict resolution"); + git.add().addFilepattern("a").call(); + RevCommit commitC3S = git.commit().setMessage("Child 3 on second bug - resolve merge conflict") + .call(); + + // Merge branch-to-merge into master + git.checkout().setName("master").call(); + mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + + // Resolve the conflict manually - merge "a" as a file + git.rm().addFilepattern("a").call(); + git.rm().addFilepattern("a/content").call(); + writeTrashFile("a", "merge conflict resolution"); + git.add().addFilepattern("a").call(); + // commit C4M + git.commit().setMessage("Child 4 on master - resolve merge conflict").call(); + + // Merge C4M (second-branch) into master (C3S) + // Conflict in virtual base should be here, but there are no conflicts in + // children + mergeResult = git.merge().include(commitC3S).call(); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.MERGED); + + } + + @Theory + public void checkFileDirMergeConflictInVirtualAncestor_ConflictInChildren_FileDir(MergeStrategy strategy) + throws Exception { + if (!strategy.equals(MergeStrategy.RECURSIVE)) { + return; + } + + Git git = Git.wrap(db); + + // master + writeTrashFile("a", "initial content"); + git.add().addFilepattern("a").call(); + RevCommit commitI = git.commit().setMessage("Initial commit").call(); + + writeTrashFile("a", "content in Ancestor 1"); + git.add().addFilepattern("a").call(); + RevCommit commitA1 = git.commit().setMessage("Ancestor 1").call(); + + writeTrashFile("a", "content in Child 1 (commited on master)"); + git.add().addFilepattern("a").call(); + // commit C1M + git.commit().setMessage("Child 1 on master").call(); + + git.checkout().setCreateBranch(true).setStartPoint(commitI).setName("branch-to-merge").call(); + + // "a" becomes a directory in A2 + git.rm().addFilepattern("a").call(); + writeTrashFile("a/content", "content in Ancestor 2 (commited on branch-to-merge)"); + git.add().addFilepattern("a/content").call(); + RevCommit commitA2 = git.commit().setMessage("Ancestor 2").call(); + + // second branch + git.checkout().setCreateBranch(true).setStartPoint(commitA1).setName("second-branch").call(); + writeTrashFile("a", "content in Child 2 (commited on second-branch)"); + git.add().addFilepattern("a").call(); + // commit C2S + git.commit().setMessage("Child 2 on second-branch").call(); + + // Merge branch-to-merge into second-branch + MergeResult mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + // Resolve the conflict manually - write a file + git.rm().addFilepattern("a").call(); + git.rm().addFilepattern("a/content").call(); + writeTrashFile("a", + "content in Child 3 (commited on second-branch) - merge conflict resolution"); + git.add().addFilepattern("a").call(); + RevCommit commitC3S = git.commit().setMessage("Child 3 on second bug - resolve merge conflict") + .call(); + + // Merge branch-to-merge into master + git.checkout().setName("master").call(); + mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + + // Resolve the conflict manually - write a file + git.rm().addFilepattern("a").call(); + git.rm().addFilepattern("a/content").call(); + writeTrashFile("a", "content in Child 4 (commited on master) - merge conflict resolution"); + git.add().addFilepattern("a").call(); + // commit C4M + git.commit().setMessage("Child 4 on master - resolve merge conflict").call(); + + // Merge C4M (second-branch) into master (C3S) + // Conflict in virtual base should be here + mergeResult = git.merge().include(commitC3S).call(); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + String expected = + "<<<<<<< HEAD\n" + "content in Child 4 (commited on master) - merge conflict resolution\n" + + "=======\n" + + "content in Child 3 (commited on second-branch) - merge conflict resolution\n" + + ">>>>>>> " + commitC3S.name() + "\n"; + assertEquals(expected, read("a")); + // Nothing was populated from the ancestors. + assertEquals( + "[a, mode:100644, stage:2, content:content in Child 4 (commited on master) - merge conflict resolution][a, mode:100644, stage:3, content:content in Child 3 (commited on second-branch) - merge conflict resolution]", + indexState(CONTENT)); + } + + /** + * Same test as above, but "a" is a dir in A1 and a file in A2 + */ + @Theory + public void checkFileDirMergeConflictInVirtualAncestor_ConflictInChildren_DirFile(MergeStrategy strategy) + throws Exception { + if (!strategy.equals(MergeStrategy.RECURSIVE)) { + return; + } + + Git git = Git.wrap(db); + + // master + writeTrashFile("a/content", "initial content"); + git.add().addFilepattern("a/content").call(); + RevCommit commitI = git.commit().setMessage("Initial commit").call(); + + writeTrashFile("a/content", "content in Ancestor 1"); + git.add().addFilepattern("a/content").call(); + RevCommit commitA1 = git.commit().setMessage("Ancestor 1").call(); + + writeTrashFile("a/content", "content in Child 1 (commited on master)"); + git.add().addFilepattern("a/content").call(); + // commit C1M + git.commit().setMessage("Child 1 on master").call(); + + git.checkout().setCreateBranch(true).setStartPoint(commitI).setName("branch-to-merge").call(); + + // "a" becomes a file in A2 + git.rm().addFilepattern("a/content").call(); + writeTrashFile("a", "content in Ancestor 2 (commited on branch-to-merge)"); + git.add().addFilepattern("a").call(); + RevCommit commitA2 = git.commit().setMessage("Ancestor 2").call(); + + // second branch + git.checkout().setCreateBranch(true).setStartPoint(commitA1).setName("second-branch").call(); + writeTrashFile("a/content", "content in Child 2 (commited on second-branch)"); + git.add().addFilepattern("a/content").call(); + // commit C2S + git.commit().setMessage("Child 2 on second-branch").call(); + + // Merge branch-to-merge into second-branch + MergeResult mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + // Resolve the conflict manually - write a file + git.rm().addFilepattern("a").call(); + git.rm().addFilepattern("a/content").call(); + deleteTrashFile("a/content"); + deleteTrashFile("a"); + writeTrashFile("a", "content in Child 3 (commited on second-branch) - merge conflict resolution"); + git.add().addFilepattern("a").call(); + RevCommit commitC3S = git.commit().setMessage("Child 3 on second bug - resolve merge conflict").call(); + + // Merge branch-to-merge into master + git.checkout().setName("master").call(); + mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + + // Resolve the conflict manually - write a file + git.rm().addFilepattern("a").call(); + git.rm().addFilepattern("a/content").call(); + deleteTrashFile("a/content"); + deleteTrashFile("a"); + writeTrashFile("a", "content in Child 4 (commited on master) - merge conflict resolution"); + git.add().addFilepattern("a").call(); + // commit C4M + git.commit().setMessage("Child 4 on master - resolve merge conflict").call(); + + // Merge C4M (second-branch) into master (C3S) + // Conflict in virtual base should be here + mergeResult = git.merge().include(commitC3S).call(); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + String expected = "<<<<<<< HEAD\n" + "content in Child 4 (commited on master) - merge conflict resolution\n" + + "=======\n" + "content in Child 3 (commited on second-branch) - merge conflict resolution\n" + + ">>>>>>> " + commitC3S.name() + "\n"; + assertEquals(expected, read("a")); + // Nothing was populated from the ancestors. + assertEquals( + "[a, mode:100644, stage:2, content:content in Child 4 (commited on master) - merge conflict resolution][a, mode:100644, stage:3, content:content in Child 3 (commited on second-branch) - merge conflict resolution]", + indexState(CONTENT)); + } + private void writeSubmodule(String path, ObjectId commit) throws IOException, ConfigInvalidException { addSubmoduleToIndex(path, commit); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/DateRevQueueTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/DateRevQueueTest.java index a9dfe15c9..852d18c35 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/DateRevQueueTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/DateRevQueueTest.java @@ -59,20 +59,26 @@ public void testInsertOutOfOrder() throws Exception { public void testInsertTie() throws Exception { final RevCommit a = parseBody(commit()); final RevCommit b = parseBody(commit(0, a)); + final RevCommit c = parseBody(commit(0, b)); + { q = create(); q.add(a); q.add(b); + q.add(c); assertCommit(a, q.next()); assertCommit(b, q.next()); + assertCommit(c, q.next()); assertNull(q.next()); } { q = create(); + q.add(c); q.add(b); q.add(a); + assertCommit(c, q.next()); assertCommit(b, q.next()); assertCommit(a, q.next()); assertNull(q.next()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevTagParseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevTagParseTest.java index b92a0726e..a3ba3d67b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevTagParseTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevTagParseTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2010, Google Inc. and others + * Copyright (C) 2008, 2020, Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -11,6 +11,7 @@ package org.eclipse.jgit.revwalk; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -18,6 +19,7 @@ import static org.junit.Assert.assertSame; import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.junit.RepositoryTestCase; @@ -117,6 +119,7 @@ public void testParseAllFields() throws Exception { assertNotNull(c.getTagName()); assertEquals(name, c.getTagName()); assertEquals("", c.getFullMessage()); + assertNull(c.getRawGpgSignature()); final PersonIdent cTagger = c.getTaggerIdent(); assertNotNull(cTagger); @@ -128,13 +131,12 @@ public void testParseAllFields() throws Exception { public void testParseOldStyleNoTagger() throws Exception { final ObjectId treeId = id("9788669ad918b6fcce64af8882fc9a81cb6aba67"); final String name = "v1.2.3.4.5"; - final String message = "test\n" // - + "\n" // - + "-----BEGIN PGP SIGNATURE-----\n" // + final String fakeSignature = "-----BEGIN PGP SIGNATURE-----\n" // + "Version: GnuPG v1.4.1 (GNU/Linux)\n" // + "\n" // + "iD8DBQBC0b9oF3Y\n" // - + "-----END PGP SIGNATURE------n"; + + "-----END PGP SIGNATURE-----"; + final String message = "test\n" + fakeSignature + '\n'; final StringBuilder body = new StringBuilder(); @@ -166,7 +168,9 @@ public void testParseOldStyleNoTagger() throws Exception { assertNotNull(c.getTagName()); assertEquals(name, c.getTagName()); assertEquals("test", c.getShortMessage()); - assertEquals(message, c.getFullMessage()); + assertEquals("test\n", c.getFullMessage()); + assertEquals(fakeSignature + '\n', + new String(c.getRawGpgSignature(), US_ASCII)); assertNull(c.getTaggerIdent()); } @@ -385,6 +389,108 @@ public void testParse_unsupportedEncoding() throws Exception { assertEquals("message\n", t.getFullMessage()); } + @Test + public void testParse_gpgSignature() throws Exception { + final String signature = "-----BEGIN PGP SIGNATURE-----\n\n" + + "wsBcBAABCAAQBQJbGB4pCRBK7hj4Ov3rIwAAdHIIAENrvz23867ZgqrmyPemBEZP\n" + + "U24B1Tlq/DWvce2buaxmbNQngKZ0pv2s8VMc11916WfTIC9EKvioatmpjduWvhqj\n" + + "znQTFyiMor30pyYsfrqFuQZvqBW01o8GEWqLg8zjf9Rf0R3LlOEw86aT8CdHRlm6\n" + + "wlb22xb8qoX4RB+LYfz7MhK5F+yLOPXZdJnAVbuyoMGRnDpwdzjL5Hj671+XJxN5\n" + + "SasRdhxkkfw/ZnHxaKEc4juMz8Nziz27elRwhOQqlTYoXNJnsV//wy5Losd7aKi1\n" + + "xXXyUpndEOmT0CIcKHrN/kbYoVL28OJaxoBuva3WYQaRrzEe3X02NMxZe9gkSqA=\n" + + "=TClh\n" + "-----END PGP SIGNATURE-----"; + ByteArrayOutputStream b = new ByteArrayOutputStream(); + b.write("object 9788669ad918b6fcce64af8882fc9a81cb6aba67\n" + .getBytes(UTF_8)); + b.write("type tree\n".getBytes(UTF_8)); + b.write("tag v1.0\n".getBytes(UTF_8)); + b.write("tagger t 1218123387 +0700\n".getBytes(UTF_8)); + b.write('\n'); + b.write("message\n".getBytes(UTF_8)); + b.write(signature.getBytes(US_ASCII)); + b.write('\n'); + + RevTag t = new RevTag(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); + try (RevWalk rw = new RevWalk(db)) { + t.parseCanonical(rw, b.toByteArray()); + } + + assertEquals("t", t.getTaggerIdent().getName()); + assertEquals("message", t.getShortMessage()); + assertEquals("message\n", t.getFullMessage()); + String gpgSig = new String(t.getRawGpgSignature(), UTF_8); + assertEquals(signature + '\n', gpgSig); + } + + @Test + public void testParse_gpgSignature2() throws Exception { + final String signature = "-----BEGIN PGP SIGNATURE-----\n\n" + + "wsBcBAABCAAQBQJbGB4pCRBK7hj4Ov3rIwAAdHIIAENrvz23867ZgqrmyPemBEZP\n" + + "U24B1Tlq/DWvce2buaxmbNQngKZ0pv2s8VMc11916WfTIC9EKvioatmpjduWvhqj\n" + + "znQTFyiMor30pyYsfrqFuQZvqBW01o8GEWqLg8zjf9Rf0R3LlOEw86aT8CdHRlm6\n" + + "wlb22xb8qoX4RB+LYfz7MhK5F+yLOPXZdJnAVbuyoMGRnDpwdzjL5Hj671+XJxN5\n" + + "SasRdhxkkfw/ZnHxaKEc4juMz8Nziz27elRwhOQqlTYoXNJnsV//wy5Losd7aKi1\n" + + "xXXyUpndEOmT0CIcKHrN/kbYoVL28OJaxoBuva3WYQaRrzEe3X02NMxZe9gkSqA=\n" + + "=TClh\n" + "-----END PGP SIGNATURE-----"; + ByteArrayOutputStream b = new ByteArrayOutputStream(); + b.write("object 9788669ad918b6fcce64af8882fc9a81cb6aba67\n" + .getBytes(UTF_8)); + b.write("type tree\n".getBytes(UTF_8)); + b.write("tag v1.0\n".getBytes(UTF_8)); + b.write("tagger t 1218123387 +0700\n".getBytes(UTF_8)); + b.write('\n'); + String message = "message\n\n" + signature.replace("xXXy", "aAAb") + + '\n'; + b.write(message.getBytes(UTF_8)); + b.write(signature.getBytes(US_ASCII)); + b.write('\n'); + + RevTag t = new RevTag(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); + try (RevWalk rw = new RevWalk(db)) { + t.parseCanonical(rw, b.toByteArray()); + } + + assertEquals("t", t.getTaggerIdent().getName()); + assertEquals("message", t.getShortMessage()); + assertEquals(message, t.getFullMessage()); + String gpgSig = new String(t.getRawGpgSignature(), UTF_8); + assertEquals(signature + '\n', gpgSig); + } + + @Test + public void testParse_gpgSignature3() throws Exception { + final String signature = "-----BEGIN PGP SIGNATURE-----\n\n" + + "wsBcBAABCAAQBQJbGB4pCRBK7hj4Ov3rIwAAdHIIAENrvz23867ZgqrmyPemBEZP\n" + + "U24B1Tlq/DWvce2buaxmbNQngKZ0pv2s8VMc11916WfTIC9EKvioatmpjduWvhqj\n" + + "znQTFyiMor30pyYsfrqFuQZvqBW01o8GEWqLg8zjf9Rf0R3LlOEw86aT8CdHRlm6\n" + + "wlb22xb8qoX4RB+LYfz7MhK5F+yLOPXZdJnAVbuyoMGRnDpwdzjL5Hj671+XJxN5\n" + + "SasRdhxkkfw/ZnHxaKEc4juMz8Nziz27elRwhOQqlTYoXNJnsV//wy5Losd7aKi1\n" + + "xXXyUpndEOmT0CIcKHrN/kbYoVL28OJaxoBuva3WYQaRrzEe3X02NMxZe9gkSqA=\n" + + "=TClh\n" + "-----END PGP SIGNATURE-----"; + ByteArrayOutputStream b = new ByteArrayOutputStream(); + b.write("object 9788669ad918b6fcce64af8882fc9a81cb6aba67\n" + .getBytes(UTF_8)); + b.write("type tree\n".getBytes(UTF_8)); + b.write("tag v1.0\n".getBytes(UTF_8)); + b.write("tagger t 1218123387 +0700\n".getBytes(UTF_8)); + b.write('\n'); + String message = "message\n\n-----BEGIN PGP SIGNATURE-----\n"; + b.write(message.getBytes(UTF_8)); + b.write(signature.getBytes(US_ASCII)); + b.write('\n'); + + RevTag t = new RevTag(id("9473095c4cb2f12aefe1db8a355fe3fafba42f67")); + try (RevWalk rw = new RevWalk(db)) { + t.parseCanonical(rw, b.toByteArray()); + } + + assertEquals("t", t.getTaggerIdent().getName()); + assertEquals("message", t.getShortMessage()); + assertEquals(message, t.getFullMessage()); + String gpgSig = new String(t.getRawGpgSignature(), UTF_8); + assertEquals(signature + '\n', gpgSig); + } + @Test public void testParse_NoMessage() throws Exception { final String msg = ""; @@ -447,7 +553,8 @@ public void testParse_GitStyleMessage() throws Exception { } @Test - public void testParse_PublicParseMethod() throws CorruptObjectException { + public void testParse_PublicParseMethod() + throws CorruptObjectException, UnsupportedEncodingException { TagBuilder src = new TagBuilder(); try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { src.setObjectId(fmt.idFor(Constants.OBJ_TREE, new byte[] {}), diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java index 803ff108f..b8b503cdc 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/FileBasedConfigTest.java @@ -13,6 +13,7 @@ import static org.eclipse.jgit.util.FileUtils.pathToString; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -178,7 +179,7 @@ public void testIncludeRelativeDotDot() final Path includedFile = createFile(CONTENT1.getBytes(UTF_8), "dir1"); final ByteArrayOutputStream bos = new ByteArrayOutputStream(); bos.write("[include]\npath=".getBytes(UTF_8)); - bos.write(("../" + includedFile.getParent().getFileName() + "/" + bos.write(("../" + parent(includedFile).getFileName() + "/" + includedFile.getFileName()).getBytes(UTF_8)); final Path file = createFile(bos.toByteArray(), "dir2"); @@ -213,7 +214,7 @@ public void testIncludeWithTilde() final Path file = createFile(bos.toByteArray(), "repo"); final FS fs = FS.DETECTED.newInstance(); - fs.setUserHome(includedFile.getParent().toFile()); + fs.setUserHome(parent(includedFile).toFile()); final FileBasedConfig config = new FileBasedConfig(file.toFile(), fs); config.load(); @@ -231,7 +232,7 @@ public void testIncludeDontInlineIncludedLinesOnSave() FileBasedConfig config = new FileBasedConfig(file.toFile(), FS.DETECTED); config.setString("include", null, "path", - ("../" + includedFile.getParent().getFileName() + "/" + ("../" + parent(includedFile).getFileName() + "/" + includedFile.getFileName())); // just by setting the include.path, it won't be included @@ -280,4 +281,10 @@ private Path createFile(byte[] content, String subdir) throws IOException { } return f; } + + private Path parent(Path file) { + Path parent = file.getParent(); + assertNotNull(parent); + return parent; + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java index 64b16f659..7d438c1dd 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/BasePackConnectionTest.java @@ -16,11 +16,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; -import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.Ref; @@ -29,18 +29,6 @@ public class BasePackConnectionTest { - @Test - public void testExtractSymRefsFromCapabilities() { - final Map symRefs = BasePackConnection - .extractSymRefsFromCapabilities( - Arrays.asList("symref=HEAD:refs/heads/main", - "symref=refs/heads/sym:refs/heads/other")); - - assertEquals(2, symRefs.size()); - assertEquals("refs/heads/main", symRefs.get("HEAD")); - assertEquals("refs/heads/other", symRefs.get("refs/heads/sym")); - } - @Test public void testUpdateWithSymRefsAdds() { final Ref mainRef = new ObjectIdRef.Unpeeled(Ref.Storage.LOOSE, @@ -230,4 +218,30 @@ public void testUpdateWithSymRefsIgnoreCircularReference() { assertThat(refMap, not(hasKey("refs/heads/sym1"))); assertThat(refMap, not(hasKey("refs/heads/sym2"))); } + + @Test + public void testUpdateWithSymRefsFillInHead() { + final String oidName = "0000000000000000000000000000000000000001"; + final Ref advertised = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, + Constants.HEAD, ObjectId.fromString(oidName)); + + final Map refMap = new HashMap<>(); + refMap.put(advertised.getName(), advertised); + + final Map symRefs = new HashMap<>(); + symRefs.put("HEAD", "refs/heads/main"); + + BasePackConnection.updateWithSymRefs(refMap, symRefs); + + assertThat(refMap, hasKey("HEAD")); + assertThat(refMap, hasKey("refs/heads/main")); + final Ref headRef = refMap.get("HEAD"); + final Ref mainRef = refMap.get("refs/heads/main"); + assertThat(headRef, instanceOf(SymbolicRef.class)); + final SymbolicRef headSymRef = (SymbolicRef) headRef; + assertEquals(Constants.HEAD, headSymRef.getName()); + assertSame(mainRef, headSymRef.getTarget()); + assertEquals(oidName, headRef.getObjectId().name()); + assertEquals(oidName, mainRef.getObjectId().name()); + } } \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java index 07c236dab..60b8098b3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PackParserTest.java @@ -29,7 +29,7 @@ import org.eclipse.jgit.errors.TooLargeObjectInPackException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser; -import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.Pack; import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; @@ -63,16 +63,16 @@ public void test1() throws IOException { try (InputStream is = new FileInputStream(packFile)) { ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); p.parse(NullProgressMonitor.INSTANCE); - PackFile file = p.getPackFile(); + Pack pack = p.getPack(); - assertTrue(file.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); - assertTrue(file.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); - assertTrue(file.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); - assertTrue(file.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); - assertTrue(file.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); - assertTrue(file.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); - assertTrue(file.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); - assertTrue(file.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); + assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"))); + assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"))); + assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"))); + assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"))); + assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); + assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"))); + assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"))); + assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"))); } } @@ -88,20 +88,20 @@ public void test2() throws IOException { try (InputStream is = new FileInputStream(packFile)) { ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is); p.parse(NullProgressMonitor.INSTANCE); - PackFile file = p.getPackFile(); + Pack pack = p.getPack(); - assertTrue(file.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); - assertTrue(file.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); - assertTrue(file.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); - assertTrue(file.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); - assertTrue(file.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); - assertTrue(file.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); - assertTrue(file.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); - assertTrue(file.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); - assertTrue(file.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); - assertTrue(file.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); - assertTrue(file.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); - assertTrue(file.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); + assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9"))); + assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6"))); + assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680"))); + assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181"))); + assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3"))); + assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6"))); + assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8"))); + assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6"))); + assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3"))); + assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e"))); + assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8"))); + assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658"))); // and lots more... } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java index 5d7f881ab..505e0088c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/PacketLineInTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, Google Inc. and others + * Copyright (C) 2009, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,8 +12,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; @@ -111,20 +112,25 @@ public void testReadString_Len0004() throws IOException { final String act = in.readString(); assertEquals("", act); assertFalse(PacketLineIn.isEnd(act)); + assertFalse(PacketLineIn.isDelimiter(act)); assertEOF(); } @Test public void testReadString_End() throws IOException { init("0000"); - assertTrue(PacketLineIn.isEnd(in.readString())); + String act = in.readString(); + assertTrue(PacketLineIn.isEnd(act)); + assertFalse(PacketLineIn.isDelimiter(act)); assertEOF(); } @Test public void testReadString_Delim() throws IOException { init("0001"); - assertTrue(PacketLineIn.isDelimiter(in.readString())); + String act = in.readString(); + assertTrue(PacketLineIn.isDelimiter(act)); + assertFalse(PacketLineIn.isEnd(act)); assertEOF(); } @@ -292,6 +298,58 @@ public void testReadACK_ERR() throws IOException { } } + // parseACKv2 + + @Test + public void testParseAckV2_NAK() throws IOException { + final ObjectId expid = ObjectId + .fromString("fcfcfb1fd94829c1a1704f894fc111d14770d34e"); + final MutableObjectId actid = new MutableObjectId(); + actid.fromString(expid.name()); + + assertSame(PacketLineIn.AckNackResult.NAK, + PacketLineIn.parseACKv2("NAK", actid)); + assertEquals(expid, actid); + } + + @Test + public void testParseAckV2_ACK() throws IOException { + final ObjectId expid = ObjectId + .fromString("fcfcfb1fd94829c1a1704f894fc111d14770d34e"); + final MutableObjectId actid = new MutableObjectId(); + + assertSame(PacketLineIn.AckNackResult.ACK_COMMON, + PacketLineIn.parseACKv2( + "ACK fcfcfb1fd94829c1a1704f894fc111d14770d34e", actid)); + assertEquals(expid, actid); + } + + @Test + public void testParseAckV2_Ready() throws IOException { + final ObjectId expid = ObjectId + .fromString("fcfcfb1fd94829c1a1704f894fc111d14770d34e"); + final MutableObjectId actid = new MutableObjectId(); + actid.fromString(expid.name()); + + assertSame(PacketLineIn.AckNackResult.ACK_READY, + PacketLineIn.parseACKv2("ready", actid)); + assertEquals(expid, actid); + } + + @Test + public void testParseAckV2_ERR() { + IOException e = assertThrows(IOException.class, () -> PacketLineIn + .parseACKv2("ERR want is not valid", new MutableObjectId())); + assertTrue(e.getMessage().contains("want is not valid")); + } + + @Test + public void testParseAckV2_Invalid() { + IOException e = assertThrows(IOException.class, + () -> PacketLineIn.parseACKv2("HELO", new MutableObjectId())); + assertTrue(e.getMessage().contains("xpected ACK/NAK")); + } + // test support private void init(String msg) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/TransferConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/TransferConfigTest.java new file mode 100644 index 000000000..d9b85fb00 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/TransferConfigTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.eclipse.jgit.lib.Config; +import org.junit.Test; + +/** + * Tests for {@link TransferConfig} parsing. + */ +public class TransferConfigTest { + + @Test + public void testParseProtocolV0() { + Config rc = new Config(); + rc.setInt("protocol", null, "version", 0); + TransferConfig tc = new TransferConfig(rc); + assertEquals(TransferConfig.ProtocolVersion.V0, tc.protocolVersion); + } + + @Test + public void testParseProtocolV1() { + Config rc = new Config(); + rc.setInt("protocol", null, "version", 1); + TransferConfig tc = new TransferConfig(rc); + assertEquals(TransferConfig.ProtocolVersion.V0, tc.protocolVersion); + } + + @Test + public void testParseProtocolV2() { + Config rc = new Config(); + rc.setInt("protocol", null, "version", 2); + TransferConfig tc = new TransferConfig(rc); + assertEquals(TransferConfig.ProtocolVersion.V2, tc.protocolVersion); + } + + @Test + public void testParseProtocolNotSet() { + Config rc = new Config(); + TransferConfig tc = new TransferConfig(rc); + assertNull(tc.protocolVersion); + } + + @Test + public void testParseProtocolUnknown() { + Config rc = new Config(); + rc.setInt("protocol", null, "version", 3); + TransferConfig tc = new TransferConfig(rc); + assertNull(tc.protocolVersion); + } + + @Test + public void testParseProtocolInvalid() { + Config rc = new Config(); + rc.setString("protocol", null, "version", "foo"); + TransferConfig tc = new TransferConfig(rc); + assertNull(tc.protocolVersion); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java index ce546e357..5045e9464 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java @@ -38,6 +38,7 @@ import org.eclipse.jgit.internal.storage.pack.CachedPack; import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; @@ -370,7 +371,8 @@ private ByteArrayInputStream uploadPackSetup(String version, ByteArrayInputStream send = linesAsInputStream(inputLines); - server.getConfig().setString("protocol", null, "version", version); + server.getConfig().setString(ConfigConstants.CONFIG_PROTOCOL_SECTION, + null, ConfigConstants.CONFIG_KEY_VERSION, version); UploadPack up = new UploadPack(server); if (postConstructionSetup != null) { postConstructionSetup.accept(up); @@ -435,8 +437,9 @@ private ByteArrayInputStream uploadPackV2( Consumer postConstructionSetup, String... inputLines) throws Exception { - ByteArrayInputStream recvStream = - uploadPackSetup("2", postConstructionSetup, inputLines); + ByteArrayInputStream recvStream = uploadPackSetup( + TransferConfig.ProtocolVersion.V2.version(), + postConstructionSetup, inputLines); PacketLineIn pckIn = new PacketLineIn(recvStream); // drain capabilities @@ -476,9 +479,11 @@ public void onFetch(FetchV2Request req) { @Test public void testV2Capabilities() throws Exception { TestV2Hook hook = new TestV2Hook(); - ByteArrayInputStream recvStream = uploadPackSetup( "2", - (UploadPack up) -> {up.setProtocolV2Hook(hook);}, - PacketLineIn.end()); + ByteArrayInputStream recvStream = uploadPackSetup( + TransferConfig.ProtocolVersion.V2.version(), + (UploadPack up) -> { + up.setProtocolV2Hook(hook); + }, PacketLineIn.end()); PacketLineIn pckIn = new PacketLineIn(recvStream); assertThat(hook.capabilitiesRequest, notNullValue()); assertThat(pckIn.readString(), is("version 2")); @@ -498,8 +503,9 @@ public void testV2Capabilities() throws Exception { private void checkAdvertisedIfAllowed(String configSection, String configName, String fetchCapability) throws Exception { server.getConfig().setBoolean(configSection, null, configName, true); - ByteArrayInputStream recvStream = - uploadPackSetup("2", null, PacketLineIn.end()); + ByteArrayInputStream recvStream = uploadPackSetup( + TransferConfig.ProtocolVersion.V2.version(), null, + PacketLineIn.end()); PacketLineIn pckIn = new PacketLineIn(recvStream); assertThat(pckIn.readString(), is("version 2")); @@ -522,8 +528,9 @@ private void checkAdvertisedIfAllowed(String configSection, String configName, private void checkUnadvertisedIfUnallowed(String configSection, String configName, String fetchCapability) throws Exception { server.getConfig().setBoolean(configSection, null, configName, false); - ByteArrayInputStream recvStream = - uploadPackSetup("2", null, PacketLineIn.end()); + ByteArrayInputStream recvStream = uploadPackSetup( + TransferConfig.ProtocolVersion.V2.version(), null, + PacketLineIn.end()); PacketLineIn pckIn = new PacketLineIn(recvStream); assertThat(pckIn.readString(), is("version 2")); @@ -574,8 +581,9 @@ public void testV2CapabilitiesAdvertiseSidebandAll() throws Exception { public void testV2CapabilitiesRefInWantNotAdvertisedIfAdvertisingForbidden() throws Exception { server.getConfig().setBoolean("uploadpack", null, "allowrefinwant", true); server.getConfig().setBoolean("uploadpack", null, "advertiserefinwant", false); - ByteArrayInputStream recvStream = - uploadPackSetup("2", null, PacketLineIn.end()); + ByteArrayInputStream recvStream = uploadPackSetup( + TransferConfig.ProtocolVersion.V2.version(), null, + PacketLineIn.end()); PacketLineIn pckIn = new PacketLineIn(recvStream); assertThat(pckIn.readString(), is("version 2")); @@ -739,7 +747,10 @@ public void testV2LsRefsServerOptions() throws Exception { PacketLineIn.end() }; TestV2Hook testHook = new TestV2Hook(); - uploadPackSetup("2", (UploadPack up) -> {up.setProtocolV2Hook(testHook);}, lines); + uploadPackSetup(TransferConfig.ProtocolVersion.V2.version(), + (UploadPack up) -> { + up.setProtocolV2Hook(testHook); + }, lines); LsRefsV2Request req = testHook.lsRefsRequest; assertEquals(2, req.getServerOptions().size()); @@ -1559,7 +1570,10 @@ public void testV2FetchServerOptions() throws Exception { PacketLineIn.end() }; TestV2Hook testHook = new TestV2Hook(); - uploadPackSetup("2", (UploadPack up) -> {up.setProtocolV2Hook(testHook);}, lines); + uploadPackSetup(TransferConfig.ProtocolVersion.V2.version(), + (UploadPack up) -> { + up.setProtocolV2Hook(testHook); + }, lines); FetchV2Request req = testHook.fetchRequest; assertNotNull(req); @@ -2253,7 +2267,9 @@ public void testGetPeerAgentProtocolV0() throws Exception { @Test public void testGetPeerAgentProtocolV2() throws Exception { - server.getConfig().setString("protocol", null, "version", "2"); + server.getConfig().setString(ConfigConstants.CONFIG_PROTOCOL_SECTION, + null, ConfigConstants.CONFIG_KEY_VERSION, + TransferConfig.ProtocolVersion.V2.version()); RevCommit one = remote.commit().message("1").create(); remote.update("one", one); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/IOTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/IOTest.java new file mode 100644 index 000000000..10a858fbf --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/IOTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.junit.Test; + +public class IOTest { + + private static final byte[] DATA = "abcdefghijklmnopqrstuvwxyz" + .getBytes(StandardCharsets.US_ASCII); + + private byte[] initBuffer(int size) { + byte[] buffer = new byte[size]; + for (int i = 0; i < size; i++) { + buffer[i] = (byte) ('0' + (i % 10)); + } + return buffer; + } + + private int read(byte[] buffer, int from) throws IOException { + try (InputStream in = new ByteArrayInputStream(DATA)) { + return IO.readFully(in, buffer, from); + } + } + + @Test + public void readFullyBufferShorter() throws Exception { + byte[] buffer = initBuffer(9); + int length = read(buffer, 0); + assertEquals(buffer.length, length); + assertArrayEquals(buffer, Arrays.copyOfRange(DATA, 0, length)); + } + + @Test + public void readFullyBufferLonger() throws Exception { + byte[] buffer = initBuffer(50); + byte[] initial = Arrays.copyOf(buffer, buffer.length); + int length = read(buffer, 0); + assertEquals(DATA.length, length); + assertArrayEquals(Arrays.copyOfRange(buffer, 0, length), DATA); + assertArrayEquals(Arrays.copyOfRange(buffer, length, buffer.length), + Arrays.copyOfRange(initial, length, initial.length)); + } + + @Test + public void readFullyBufferShorterOffset() throws Exception { + byte[] buffer = initBuffer(9); + byte[] initial = Arrays.copyOf(buffer, buffer.length); + int length = read(buffer, 6); + assertEquals(3, length); + assertArrayEquals(Arrays.copyOfRange(buffer, 0, 6), + Arrays.copyOfRange(initial, 0, 6)); + assertArrayEquals(Arrays.copyOfRange(buffer, 6, buffer.length), + Arrays.copyOfRange(DATA, 0, 3)); + } + + @Test + public void readFullyBufferLongerOffset() throws Exception { + byte[] buffer = initBuffer(50); + byte[] initial = Arrays.copyOf(buffer, buffer.length); + int length = read(buffer, 40); + assertEquals(10, length); + assertArrayEquals(Arrays.copyOfRange(buffer, 0, 40), + Arrays.copyOfRange(initial, 0, 40)); + assertArrayEquals(Arrays.copyOfRange(buffer, 40, buffer.length), + Arrays.copyOfRange(DATA, 0, 10)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java index 4e65ca7a4..01dcde29b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/TemporaryBufferTest.java @@ -406,4 +406,69 @@ public void testHeapWithEstimatedSize() throws IOException { } } } + + @Test + public void testHeapToByteArrayWithLimit() throws IOException { + int sz = 2 * Block.SZ; + try (TemporaryBuffer b = new TemporaryBuffer.Heap(sz / 2, sz)) { + for (int i = 0; i < sz; i++) { + b.write('a' + i % 26); + } + byte[] prefix = b.toByteArray(5); + assertEquals(5, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + prefix = b.toByteArray(Block.SZ + 37); + assertEquals(Block.SZ + 37, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + prefix = b.toByteArray(sz); + assertEquals(sz, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + prefix = b.toByteArray(sz + 37); + assertEquals(sz, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + } + } + + @Test + public void testFileToByteArrayWithLimit() throws IOException { + @SuppressWarnings("resource") // Buffer is explicitly destroyed in finally block + TemporaryBuffer b = new TemporaryBuffer.LocalFile(null, 2 * Block.SZ); + int sz = 3 * Block.SZ; + try { + for (int i = 0; i < sz; i++) { + b.write('a' + i % 26); + } + b.close(); + byte[] prefix = b.toByteArray(5); + assertEquals(5, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + prefix = b.toByteArray(Block.SZ + 37); + assertEquals(Block.SZ + 37, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + prefix = b.toByteArray(sz); + assertEquals(sz, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + prefix = b.toByteArray(sz + 37); + assertEquals(sz, prefix.length); + for (int i = 0; i < prefix.length; i++) { + assertEquals('a' + i % 26, prefix[i]); + } + } finally { + b.destroy(); + } + } } diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 538c6f7bf..d389ac588 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,38 +1,56 @@ - - + + - - + + - - + + - - - - - - - - + + - - + + - - + + - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/BUILD b/org.eclipse.jgit/BUILD index 208337224..04873b0c7 100644 --- a/org.eclipse.jgit/BUILD +++ b/org.eclipse.jgit/BUILD @@ -38,7 +38,7 @@ genrule( "cd $$TMP", "unzip -q $$ROOT/$<", "echo \"Implementation-Version: $$GEN_VERSION\n$$(cat META-INF/MANIFEST.MF)\" > META-INF/MANIFEST.MF", - "find . -exec touch '{}' ';'", + "find . -exec touch -t 198001010000 '{}' ';'", "zip -Xqr $$ROOT/$@ .", "rm -rf $$TMP", ]), diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index 403b8eed0..d755f8362 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -72,11 +72,8 @@ Export-Package: org.eclipse.jgit.annotations;version="6.0.0", org.eclipse.jgit.http.test", org.eclipse.jgit.internal.fsck;version="6.0.0"; x-friends:="org.eclipse.jgit.test", - org.eclipse.jgit.internal.ketch;version="6.0.0"; - x-friends:="org.eclipse.jgit.junit, - org.eclipse.jgit.test, - org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.revwalk;version="6.0.0";x-internal:=true, + org.eclipse.jgit.internal.revwalk;version="6.0.0"; + x-friends:="org.eclipse.jgit.test", org.eclipse.jgit.internal.storage.dfs;version="6.0.0"; x-friends:="org.eclipse.jgit.test, org.eclipse.jgit.http.server, @@ -104,10 +101,6 @@ Export-Package: org.eclipse.jgit.annotations;version="6.0.0", org.eclipse.jgit.junit, org.eclipse.jgit.test, org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.reftree;version="6.0.0"; - x-friends:="org.eclipse.jgit.junit, - org.eclipse.jgit.test, - org.eclipse.jgit.pgm", org.eclipse.jgit.internal.submodule;version="6.0.0";x-internal:=true, org.eclipse.jgit.internal.transport.connectivity;version="6.0.0"; x-friends:="org.eclipse.jgit.test", diff --git a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF index e1904d5d3..134cbacbe 100644 --- a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF @@ -5,4 +5,3 @@ Bundle-SymbolicName: org.eclipse.jgit.source Bundle-Vendor: Eclipse.org - JGit Bundle-Version: 6.0.0.qualifier Eclipse-SourceBundle: org.eclipse.jgit;version="6.0.0.qualifier";roots="." - diff --git a/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml b/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml index 2efbb9c1a..73a668556 100644 --- a/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml +++ b/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml @@ -9,6 +9,12 @@ + + + + + + @@ -19,8 +25,26 @@ - - + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 12902b900..c00203dd0 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -30,6 +30,8 @@ badEntryDelimiter=Bad entry delimiter badEntryName=Bad entry name: {0} badEscape=Bad escape: {0} badGroupHeader=Bad group header +badIgnorePattern=Cannot parse .gitignore pattern ''{0}'' +badIgnorePatternFull=File {0} line {1}: cannot parse pattern ''{2}'': {3} badObjectType=Bad object type: {0} badRef=Bad ref: {0}: {1} badSectionEntry=Bad section entry: {0} @@ -233,6 +235,7 @@ downloadCancelled=Download cancelled downloadCancelledDuringIndexing=Download cancelled during indexing duplicateAdvertisementsOf=duplicate advertisements of {0} duplicateRef=Duplicate ref: {0} +duplicateRefAttribute=Duplicate ref attribute: {0} duplicateRemoteRefUpdateIsIllegal=Duplicate remote ref update is illegal. Affected remote name: {0} duplicateStagesNotAllowed=Duplicate stages not allowed eitherGitDirOrWorkTreeRequired=One of setGitDir or setWorkTree must be called. @@ -310,6 +313,10 @@ headRequiredToStash=HEAD required to stash local changes hoursAgo={0} hours ago httpConfigCannotNormalizeURL=Cannot normalize URL path {0}: too many .. segments httpConfigInvalidURL=Cannot parse URL from subsection http.{0} in git config; ignored. +httpFactoryInUse=Changing the HTTP connection factory after an HTTP connection has already been opened is not allowed. +httpPreAuthTooLate=HTTP Basic preemptive authentication cannot be set once an HTTP connection has already been opened. +httpUserInfoDecodeError=Cannot decode user info from URL {}; ignored. +httpWrongConnectionType=Wrong connection type: expected {0}, got {1}. hugeIndexesAreNotSupportedByJgitYet=Huge indexes are not supported by jgit, yet hunkBelongsToAnotherFile=Hunk belongs to another file hunkDisconnectedFromFile=Hunk disconnected from file @@ -563,6 +570,7 @@ refNotResolved=Ref {0} cannot be resolved reftableDirExists=reftable dir exists and is nonempty reftableRecordsMustIncrease=records must be increasing: last {0}, this {1} refUpdateReturnCodeWas=RefUpdate return code was: {0} +remoteBranchNotFound=Remote branch ''{0}'' not found in upstream origin remoteConfigHasNoURIAssociated=Remote config "{0}" has no URIs associated remoteDoesNotHaveSpec=Remote does not have {0} available for fetch. remoteDoesNotSupportSmartHTTPPush=remote does not support smart HTTP push @@ -617,7 +625,9 @@ shortCompressedStreamAt=Short compressed stream at {0} shortReadOfBlock=Short read of block. shortReadOfOptionalDIRCExtensionExpectedAnotherBytes=Short read of optional DIRC extension {0}; expected another {1} bytes within the section. shortSkipOfBlock=Short skip of block. -signingNotSupportedOnTag=Signing isn't supported on tag operations yet. +signatureVerificationError=Signature verification failed +signatureVerificationUnavailable=No signature verifier registered +signedTagMessageNoLf=A non-empty message of a signed tag must end in LF. signingServiceUnavailable=Signing service is not available similarityScoreMustBeWithinBounds=Similarity score must be between 0 and 100. skipMustBeNonNegative=skip must be >= 0 @@ -758,6 +768,13 @@ uriNotFoundWithMessage={0} not found: {1} URINotSupported=URI not supported: {0} userConfigInvalid=Git config in the user's home directory {0} is invalid {1} validatingGitModules=Validating .gitmodules files +verifySignatureBad=BAD signature from "{0}" +verifySignatureExpired=Expired signature from "{0}" +verifySignatureGood=Good signature from "{0}" +verifySignatureIssuer=issuer "{0}" +verifySignatureKey=using key {0} +verifySignatureMade=Signature made {0} +verifySignatureTrust=[{0}] walkFailure=Walk failure. wantNoSpaceWithCapabilities=No space between oid and first capability in first want line wantNotValid=want {0} not valid diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties deleted file mode 100644 index 1fbb7cb3b..000000000 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties +++ /dev/null @@ -1,13 +0,0 @@ -accepted=accepted. -cannotFetchFromLocalReplica=cannot fetch from LocalReplica -failed=failed! -invalidFollowerUri=invalid follower URI -leaderFailedToStore=leader failed to store -localReplicaRequired=LocalReplica instance is required -mismatchedTxnNamespace=mismatched txnNamespace; expected {0} found {1} -outsideTxnNamespace=ref {0} is outside of txnNamespace {1} -proposingUpdates=Proposing updates -queuedProposalFailedToApply=queued proposal failed to apply -starting=starting! -unsupportedVoterCount=unsupported voter count {0}, expected one of {1} -waitingForQueue=Waiting for queue diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java index 2c01c19c5..fdf8b80cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; @@ -202,7 +201,7 @@ public FormatEntry(Format format, int refcnt) { * Available archival formats (corresponding to values for * the --format= option) */ - private static final ConcurrentMap formats = + private static final Map formats = new ConcurrentHashMap<>(); /** @@ -215,7 +214,7 @@ public FormatEntry(Format format, int refcnt) { * @param newValue value to be associated with the key (null to remove). * @return true if the value was replaced */ - private static boolean replace(ConcurrentMap map, + private static boolean replace(Map map, K key, V oldValue, V newValue) { if (oldValue == null && newValue == null) // Nothing to do. return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java index aba86fc36..cf7bc1f26 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java @@ -297,6 +297,7 @@ private FetchResult fetch(Repository clonedRepo, URIish u) command.setTagOpt( fetchAll ? TagOpt.FETCH_TAGS : TagOpt.AUTO_FOLLOW); } + command.setInitialBranch(branch); configure(command); return command.call(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index b4f717503..31f6a31c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -47,6 +47,7 @@ import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.GpgConfig; import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.GpgObjectSigner; import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; @@ -120,6 +121,8 @@ public class CommitCommand extends GitCommand { private GpgSigner gpgSigner; + private GpgConfig gpgConfig; + private CredentialsProvider credentialsProvider; /** @@ -247,8 +250,18 @@ public RevCommit call() throws GitAPIException, AbortedByHookException, throw new ServiceUnavailableException( JGitText.get().signingServiceUnavailable); } - gpgSigner.sign(commit, signingKey, committer, - credentialsProvider); + if (gpgSigner instanceof GpgObjectSigner) { + ((GpgObjectSigner) gpgSigner).signObject(commit, + signingKey, committer, credentialsProvider, + gpgConfig); + } else { + if (gpgConfig.getKeyFormat() != GpgFormat.OPENPGP) { + throw new UnsupportedSigningFormatException(JGitText + .get().onlyOpenPgpSupportedForSigning); + } + gpgSigner.sign(commit, signingKey, committer, + credentialsProvider); + } } ObjectId commitId = odi.insert(commit); @@ -576,7 +589,9 @@ private void processOptions(RepositoryState state, RevWalk rw) // an explicit message throw new NoMessageException(JGitText.get().commitMessageNotSpecified); - GpgConfig gpgConfig = new GpgConfig(repo.getConfig()); + if (gpgConfig == null) { + gpgConfig = new GpgConfig(repo.getConfig()); + } if (signCommit == null) { signCommit = gpgConfig.isSignCommits() ? Boolean.TRUE : Boolean.FALSE; @@ -585,10 +600,6 @@ private void processOptions(RepositoryState state, RevWalk rw) signingKey = gpgConfig.getSigningKey(); } if (gpgSigner == null) { - if (gpgConfig.getKeyFormat() != GpgFormat.OPENPGP) { - throw new UnsupportedSigningFormatException( - JGitText.get().onlyOpenPgpSupportedForSigning); - } gpgSigner = GpgSigner.getDefault(); } } @@ -972,6 +983,36 @@ public CommitCommand setSign(Boolean sign) { return this; } + /** + * Sets the {@link GpgSigner} to use if the commit is to be signed. + * + * @param signer + * to use; if {@code null}, the default signer will be used + * @return {@code this} + * @since 5.11 + */ + public CommitCommand setGpgSigner(GpgSigner signer) { + checkCallable(); + this.gpgSigner = signer; + return this; + } + + /** + * Sets an external {@link GpgConfig} to use. Whether it will be used is at + * the discretion of the {@link #setGpgSigner(GpgSigner)}. + * + * @param config + * to set; if {@code null}, the config will be loaded from the + * git config of the repository + * @return {@code this} + * @since 5.11 + */ + public CommitCommand setGpgConfig(GpgConfig config) { + checkCallable(); + this.gpgConfig = config; + return this; + } + /** * Sets a {@link CredentialsProvider} * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java index 033dd60c3..90c1515b0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java @@ -74,6 +74,8 @@ public class FetchCommand extends TransportCommand { private boolean isForceUpdate; + private String initialBranch; + /** * Callback for status of fetch operation. * @@ -209,7 +211,7 @@ public FetchResult call() throws GitAPIException, InvalidRemoteException, transport.setFetchThin(thin); configure(transport); FetchResult result = transport.fetch(monitor, - applyOptions(refSpecs)); + applyOptions(refSpecs), initialBranch); if (!repo.isBare()) { fetchSubmodules(result); } @@ -487,6 +489,24 @@ public FetchCommand setTagOpt(TagOpt tagOpt) { return this; } + /** + * Set the initial branch + * + * @param branch + * the initial branch to check out when cloning the repository. + * Can be specified as ref name (refs/heads/master), + * branch name (master) or tag name + * (v1.2.3). The default is to use the branch + * pointed to by the cloned repository's HEAD and can be + * requested by passing {@code null} or HEAD. + * @return {@code this} + * @since 5.11 + */ + public FetchCommand setInitialBranch(String branch) { + this.initialBranch = branch; + return this; + } + /** * Register a progress callback. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 64314772b..3b3e10e7b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Christian Halstrick - * Copyright (C) 2010, Chris Aniszczyk and others + * Copyright (C) 2010, 2021 Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -772,6 +772,16 @@ public RemoteSetUrlCommand remoteSetUrl() { return new RemoteSetUrlCommand(repo); } + /** + * Return a command to verify signatures of tags or commits. + * + * @return a {@link VerifySignatureCommand} + * @since 5.11 + */ + public VerifySignatureCommand verifySignature() { + return new VerifySignatureCommand(repo); + } + /** * Get repository * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java index 41fcf29ed..240290f4f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java @@ -15,12 +15,16 @@ import java.util.concurrent.Callable; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryBuilder; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; /** @@ -38,6 +42,8 @@ public class InitCommand implements Callable { private FS fs; + private String initialBranch; + /** * {@inheritDoc} *

    @@ -87,11 +93,16 @@ public Git call() throws GitAPIException { builder.setWorkTree(new File(dStr)); } } + builder.setInitialBranch(StringUtils.isEmptyOrNull(initialBranch) + ? SystemReader.getInstance().getUserConfig().getString( + ConfigConstants.CONFIG_INIT_SECTION, null, + ConfigConstants.CONFIG_KEY_DEFAULT_BRANCH) + : initialBranch); Repository repository = builder.build(); if (!repository.getObjectDatabase().exists()) repository.create(bare); return new Git(repository, true); - } catch (IOException e) { + } catch (IOException | ConfigInvalidException e) { throw new JGitInternalException(e.getMessage(), e); } } @@ -184,4 +195,23 @@ public InitCommand setFs(FS fs) { this.fs = fs; return this; } + + /** + * Set the initial branch of the new repository. If not specified + * ({@code null} or empty), fall back to the default name (currently + * master). + * + * @param branch + * initial branch name of the new repository + * @return {@code this} + * @throws InvalidRefNameException + * if the branch name is not valid + * + * @since 5.11 + */ + public InitCommand setInitialBranch(String branch) + throws InvalidRefNameException { + this.initialBranch = branch; + return this; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java index a4ca30909..0c691062f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, Christoph Brill and others + * Copyright (C) 2011, 2020 Christoph Brill and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -164,7 +164,7 @@ private Map execute() throws GitAPIException, refSpecs.add(new RefSpec("refs/heads/*:refs/remotes/origin/*")); //$NON-NLS-1$ Collection refs; Map refmap = new HashMap<>(); - try (FetchConnection fc = transport.openFetch()) { + try (FetchConnection fc = transport.openFetch(refSpecs)) { refs = fc.getRefs(); if (refSpecs.isEmpty()) for (Ref r : refs) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 6678af163..836175dce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -67,6 +67,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.merge.MergeStrategy; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevSort; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.submodule.SubmoduleWalk.IgnoreSubmoduleMode; @@ -1137,15 +1138,19 @@ else if (!isInteractive() && walk.isMergedInto(headCommit, upstream)) { private List calculatePickList(RevCommit headCommit) throws GitAPIException, NoHeadException, IOException { - Iterable commitsToUse; - try (Git git = new Git(repo)) { - LogCommand cmd = git.log().addRange(upstreamCommit, headCommit); - commitsToUse = cmd.call(); - } List cherryPickList = new ArrayList<>(); - for (RevCommit commit : commitsToUse) { - if (preserveMerges || commit.getParentCount() == 1) - cherryPickList.add(commit); + try (RevWalk r = new RevWalk(repo)) { + r.sort(RevSort.TOPO_KEEP_BRANCH_TOGETHER, true); + r.sort(RevSort.COMMIT_TIME_DESC, true); + r.markUninteresting(r.lookupCommit(upstreamCommit)); + r.markStart(r.lookupCommit(headCommit)); + Iterator commitsToUse = r.iterator(); + while (commitsToUse.hasNext()) { + RevCommit commit = commitsToUse.next(); + if (preserveMerges || commit.getParentCount() == 1) { + cherryPickList.add(commit); + } + } } Collections.reverse(cherryPickList); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java index 9a328a6ea..58c18b38d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013 Chris Aniszczyk and others + * Copyright (C) 2010, 2020 Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -18,8 +18,14 @@ import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.NoHeadException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.eclipse.jgit.api.errors.ServiceUnavailableException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.GpgObjectSigner; +import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; @@ -31,6 +37,7 @@ import org.eclipse.jgit.lib.TagBuilder; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.CredentialsProvider; /** * Create/update an annotated tag object or a simple unannotated tag @@ -56,6 +63,7 @@ * >Git documentation about Tag */ public class TagCommand extends GitCommand { + private RevObject id; private String name; @@ -64,11 +72,19 @@ public class TagCommand extends GitCommand { private PersonIdent tagger; - private boolean signed; + private Boolean signed; private boolean forceUpdate; - private boolean annotated = true; + private Boolean annotated; + + private String signingKey; + + private GpgConfig gpgConfig; + + private GpgObjectSigner gpgSigner; + + private CredentialsProvider credentialsProvider; /** *

    Constructor for TagCommand.

    @@ -77,6 +93,7 @@ public class TagCommand extends GitCommand { */ protected TagCommand(Repository repo) { super(repo); + this.credentialsProvider = CredentialsProvider.getDefault(); } /** @@ -108,10 +125,7 @@ public Ref call() throws GitAPIException, ConcurrentRefUpdateException, id = revWalk.parseCommit(objectId); } - if (!annotated) { - if (message != null || tagger != null) - throw new JGitInternalException( - JGitText.get().messageAndTaggerNotAllowedInUnannotatedTags); + if (!isAnnotated()) { return updateTagRef(id, revWalk, name, "SimpleTag[" + name + " : " + id //$NON-NLS-1$ //$NON-NLS-2$ + "]"); //$NON-NLS-1$ @@ -124,6 +138,11 @@ public Ref call() throws GitAPIException, ConcurrentRefUpdateException, newTag.setTagger(tagger); newTag.setObjectId(id); + if (gpgSigner != null) { + gpgSigner.signObject(newTag, signingKey, tagger, + credentialsProvider, gpgConfig); + } + // write the tag object try (ObjectInserter inserter = repo.newObjectInserter()) { ObjectId tagId = inserter.insert(newTag); @@ -158,9 +177,17 @@ private Ref updateTagRef(ObjectId tagId, RevWalk revWalk, throw new ConcurrentRefUpdateException( JGitText.get().couldNotLockHEAD, tagRef.getRef(), updateResult); + case NO_CHANGE: + if (forceUpdate) { + return repo.exactRef(refName); + } + throw new RefAlreadyExistsException(MessageFormat + .format(JGitText.get().tagAlreadyExists, newTagToString), + updateResult); case REJECTED: throw new RefAlreadyExistsException(MessageFormat.format( - JGitText.get().tagAlreadyExists, newTagToString)); + JGitText.get().tagAlreadyExists, newTagToString), + updateResult); default: throw new JGitInternalException(MessageFormat.format( JGitText.get().updatingRefFailed, refName, newTagToString, @@ -177,20 +204,60 @@ private Ref updateTagRef(ObjectId tagId, RevWalk revWalk, * * @throws InvalidTagNameException * if the tag name is null or invalid - * @throws UnsupportedOperationException - * if the tag is signed (not supported yet) + * @throws ServiceUnavailableException + * if the tag should be signed but no signer can be found + * @throws UnsupportedSigningFormatException + * if the tag should be signed but {@code gpg.format} is not + * {@link GpgFormat#OPENPGP} */ private void processOptions(RepositoryState state) - throws InvalidTagNameException { - if (tagger == null && annotated) - tagger = new PersonIdent(repo); - if (name == null || !Repository.isValidRefName(Constants.R_TAGS + name)) + throws InvalidTagNameException, ServiceUnavailableException, + UnsupportedSigningFormatException { + if (name == null + || !Repository.isValidRefName(Constants.R_TAGS + name)) { throw new InvalidTagNameException( MessageFormat.format(JGitText.get().tagNameInvalid, name == null ? "" : name)); //$NON-NLS-1$ - if (signed) - throw new UnsupportedOperationException( - JGitText.get().signingNotSupportedOnTag); + } + if (!isAnnotated()) { + if ((message != null && !message.isEmpty()) || tagger != null) { + throw new JGitInternalException(JGitText + .get().messageAndTaggerNotAllowedInUnannotatedTags); + } + } else { + if (tagger == null) { + tagger = new PersonIdent(repo); + } + // Figure out whether to sign. + if (!(Boolean.FALSE.equals(signed) && signingKey == null)) { + if (gpgConfig == null) { + gpgConfig = new GpgConfig(repo.getConfig()); + } + boolean doSign = isSigned() || gpgConfig.isSignAllTags(); + if (!Boolean.TRUE.equals(annotated) && !doSign) { + doSign = gpgConfig.isSignAnnotated(); + } + if (doSign) { + if (signingKey == null) { + signingKey = gpgConfig.getSigningKey(); + } + if (gpgSigner == null) { + GpgSigner signer = GpgSigner.getDefault(); + if (!(signer instanceof GpgObjectSigner)) { + throw new ServiceUnavailableException( + JGitText.get().signingServiceUnavailable); + } + gpgSigner = (GpgObjectSigner) signer; + } + // The message of a signed tag must end in a newline because + // the signature will be appended. + if (message != null && !message.isEmpty() + && !message.endsWith("\n")) { //$NON-NLS-1$ + message += '\n'; + } + } + } + } } /** @@ -238,24 +305,61 @@ public TagCommand setMessage(String message) { } /** - * Whether this tag is signed + * Whether {@link #setSigned(boolean) setSigned(true)} has been called or + * whether a {@link #setSigningKey(String) signing key ID} has been set; + * i.e., whether -s or -u was specified explicitly. * * @return whether the tag is signed */ public boolean isSigned() { - return signed; + return Boolean.TRUE.equals(signed) || signingKey != null; } /** * If set to true the Tag command creates a signed tag object. This - * corresponds to the parameter -s on the command line. + * corresponds to the parameter -s (--sign or --no-sign) on the command + * line. + *

    + * If {@code true}, the tag will be a signed annotated tag. + *

    * * @param signed - * a boolean. + * whether to sign * @return {@code this} */ public TagCommand setSigned(boolean signed) { - this.signed = signed; + checkCallable(); + this.signed = Boolean.valueOf(signed); + return this; + } + + /** + * Sets the {@link GpgSigner} to use if the commit is to be signed. + * + * @param signer + * to use; if {@code null}, the default signer will be used + * @return {@code this} + * @since 5.11 + */ + public TagCommand setGpgSigner(GpgObjectSigner signer) { + checkCallable(); + this.gpgSigner = signer; + return this; + } + + /** + * Sets an external {@link GpgConfig} to use. Whether it will be used is at + * the discretion of the {@link #setGpgSigner(GpgObjectSigner)}. + * + * @param config + * to set; if {@code null}, the config will be loaded from the + * git config of the repository + * @return {@code this} + * @since 5.11 + */ + public TagCommand setGpgConfig(GpgConfig config) { + checkCallable(); + this.gpgConfig = config; return this; } @@ -268,6 +372,7 @@ public TagCommand setSigned(boolean signed) { * @return {@code this} */ public TagCommand setTagger(PersonIdent tagger) { + checkCallable(); this.tagger = tagger; return this; } @@ -291,14 +396,15 @@ public RevObject getObjectId() { } /** - * Sets the object id of the tag. If the object id is null, the commit - * pointed to from HEAD will be used. + * Sets the object id of the tag. If the object id is {@code null}, the + * commit pointed to from HEAD will be used. * * @param id * a {@link org.eclipse.jgit.revwalk.RevObject} object. * @return {@code this} */ public TagCommand setObjectId(RevObject id) { + checkCallable(); this.id = id; return this; } @@ -321,6 +427,7 @@ public boolean isForceUpdate() { * @return {@code this} */ public TagCommand setForceUpdate(boolean forceUpdate) { + checkCallable(); this.forceUpdate = forceUpdate; return this; } @@ -334,18 +441,77 @@ public TagCommand setForceUpdate(boolean forceUpdate) { * @since 3.0 */ public TagCommand setAnnotated(boolean annotated) { - this.annotated = annotated; + checkCallable(); + this.annotated = Boolean.valueOf(annotated); return this; } /** - * Whether this will create an annotated command + * Whether this will create an annotated tag. * * @return true if this command will create an annotated tag (default is * true) * @since 3.0 */ public boolean isAnnotated() { - return annotated; + boolean setExplicitly = Boolean.TRUE.equals(annotated) || isSigned(); + if (setExplicitly) { + return true; + } + // Annotated at default (not set explicitly) + return annotated == null; } + + /** + * Sets the signing key. + *

    + * Per spec of {@code user.signingKey}: this will be sent to the GPG program + * as is, i.e. can be anything supported by the GPG program. + *

    + *

    + * Note, if none was set or {@code null} is specified a default will be + * obtained from the configuration. + *

    + *

    + * If set to a non-{@code null} value, the tag will be a signed annotated + * tag. + *

    + * + * @param signingKey + * signing key; {@code null} allowed + * @return {@code this} + * @since 5.11 + */ + public TagCommand setSigningKey(String signingKey) { + checkCallable(); + this.signingKey = signingKey; + return this; + } + + /** + * Retrieves the signing key ID. + * + * @return the key ID set, or {@code null} if none is set + * @since 5.11 + */ + public String getSigningKey() { + return signingKey; + } + + /** + * Sets a {@link CredentialsProvider} + * + * @param credentialsProvider + * the provider to use when querying for credentials (eg., during + * signing) + * @return {@code this} + * @since 5.11 + */ + public TagCommand setCredentialsProvider( + CredentialsProvider credentialsProvider) { + checkCallable(); + this.credentialsProvider = credentialsProvider; + return this; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java new file mode 100644 index 000000000..21cddf75b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api; + +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.revwalk.RevObject; + +/** + * A {@code VerificationResult} describes the outcome of a signature + * verification. + * + * @see VerifySignatureCommand + * + * @since 5.11 + */ +public interface VerificationResult { + + /** + * If an error occurred during signature verification, this retrieves the + * exception. + * + * @return the exception, or {@code null} if none occurred + */ + Throwable getException(); + + /** + * Retrieves the signature verification result. + * + * @return the result, or {@code null} if none was computed + */ + GpgSignatureVerifier.SignatureVerification getVerification(); + + /** + * Retrieves the git object of which the signature was verified. + * + * @return the git object + */ + RevObject getObject(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java new file mode 100644 index 000000000..6a2a44ea2 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java @@ -0,0 +1,307 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.ServiceUnavailableException; +import org.eclipse.jgit.api.errors.WrongObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevWalk; + +/** + * A command to verify GPG signatures on tags or commits. + * + * @since 5.11 + */ +public class VerifySignatureCommand extends GitCommand> { + + /** + * Describes what kind of objects shall be handled by a + * {@link VerifySignatureCommand}. + */ + public enum VerifyMode { + /** + * Handle any object type, ignore anything that is not a commit or tag. + */ + ANY, + /** + * Handle only commits; throw a {@link WrongObjectTypeException} for + * anything else. + */ + COMMITS, + /** + * Handle only tags; throw a {@link WrongObjectTypeException} for + * anything else. + */ + TAGS + } + + private final Set namesToCheck = new HashSet<>(); + + private VerifyMode mode = VerifyMode.ANY; + + private GpgSignatureVerifier verifier; + + private GpgConfig config; + + private boolean ownVerifier; + + /** + * Creates a new {@link VerifySignatureCommand} for the given {@link Repository}. + * + * @param repo + * to operate on + */ + public VerifySignatureCommand(Repository repo) { + super(repo); + } + + /** + * Add a name of an object (SHA-1, ref name; anything that can be + * {@link Repository#resolve(String) resolved}) to the command to have its + * signature verified. + * + * @param name + * to add + * @return {@code this} + */ + public VerifySignatureCommand addName(String name) { + checkCallable(); + namesToCheck.add(name); + return this; + } + + /** + * Add names of objects (SHA-1, ref name; anything that can be + * {@link Repository#resolve(String) resolved}) to the command to have their + * signatures verified. + * + * @param names + * to add; duplicates will be ignored + * @return {@code this} + */ + public VerifySignatureCommand addNames(String... names) { + checkCallable(); + namesToCheck.addAll(Arrays.asList(names)); + return this; + } + + /** + * Add names of objects (SHA-1, ref name; anything that can be + * {@link Repository#resolve(String) resolved}) to the command to have their + * signatures verified. + * + * @param names + * to add; duplicates will be ignored + * @return {@code this} + */ + public VerifySignatureCommand addNames(Collection names) { + checkCallable(); + namesToCheck.addAll(names); + return this; + } + + /** + * Sets the mode of operation for this command. + * + * @param mode + * the {@link VerifyMode} to set + * @return {@code this} + */ + public VerifySignatureCommand setMode(@NonNull VerifyMode mode) { + checkCallable(); + this.mode = mode; + return this; + } + + /** + * Sets the {@link GpgSignatureVerifier} to use. + * + * @param verifier + * the {@link GpgSignatureVerifier} to use, or {@code null} to + * use the default verifier + * @return {@code this} + */ + public VerifySignatureCommand setVerifier(GpgSignatureVerifier verifier) { + checkCallable(); + this.verifier = verifier; + return this; + } + + /** + * Sets an external {@link GpgConfig} to use. Whether it will be used it at + * the discretion of the {@link #setVerifier(GpgSignatureVerifier)}. + * + * @param config + * to set; if {@code null}, the config will be loaded from the + * git config of the repository + * @return {@code this} + * @since 5.11 + */ + public VerifySignatureCommand setGpgConfig(GpgConfig config) { + checkCallable(); + this.config = config; + return this; + } + + /** + * Retrieves the currently set {@link GpgSignatureVerifier}. Can be used + * after a successful {@link #call()} to get the verifier that was used. + * + * @return the {@link GpgSignatureVerifier} + */ + public GpgSignatureVerifier getVerifier() { + return verifier; + } + + /** + * {@link Repository#resolve(String) Resolves} all names added to the + * command to git objects and verifies their signature. Non-existing objects + * are ignored. + *

    + * Depending on the {@link #setMode(VerifyMode)}, only tags or commits or + * any kind of objects are allowed. + *

    + *

    + * Unsigned objects are silently skipped. + *

    + * + * @return a map of the given names to the corresponding + * {@link VerificationResult}, excluding ignored or skipped objects. + * @throws ServiceUnavailableException + * if no {@link GpgSignatureVerifier} was set and no + * {@link GpgSignatureVerifierFactory} is available + * @throws WrongObjectTypeException + * if a name resolves to an object of a type not allowed by the + * {@link #setMode(VerifyMode)} mode + */ + @Override + @NonNull + public Map call() + throws ServiceUnavailableException, WrongObjectTypeException { + checkCallable(); + setCallable(false); + Map result = new HashMap<>(); + if (verifier == null) { + GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory + .getDefault(); + if (factory == null) { + throw new ServiceUnavailableException( + JGitText.get().signatureVerificationUnavailable); + } + verifier = factory.getVerifier(); + ownVerifier = true; + } + if (config == null) { + config = new GpgConfig(repo.getConfig()); + } + try (RevWalk walk = new RevWalk(repo)) { + for (String toCheck : namesToCheck) { + ObjectId id = repo.resolve(toCheck); + if (id != null && !ObjectId.zeroId().equals(id)) { + RevObject object; + try { + object = walk.parseAny(id); + } catch (MissingObjectException e) { + continue; + } + VerificationResult verification = verifyOne(object); + if (verification != null) { + result.put(toCheck, verification); + } + } + } + } catch (IOException e) { + throw new JGitInternalException( + JGitText.get().signatureVerificationError, e); + } finally { + if (ownVerifier) { + verifier.clear(); + } + } + return result; + } + + private VerificationResult verifyOne(RevObject object) + throws WrongObjectTypeException, IOException { + int type = object.getType(); + if (VerifyMode.TAGS.equals(mode) && type != Constants.OBJ_TAG) { + throw new WrongObjectTypeException(object, Constants.OBJ_TAG); + } else if (VerifyMode.COMMITS.equals(mode) + && type != Constants.OBJ_COMMIT) { + throw new WrongObjectTypeException(object, Constants.OBJ_COMMIT); + } + if (type == Constants.OBJ_COMMIT || type == Constants.OBJ_TAG) { + try { + GpgSignatureVerifier.SignatureVerification verification = verifier + .verifySignature(object, config); + if (verification == null) { + // Not signed + return null; + } + // Create new result + return new Result(object, verification, null); + } catch (JGitInternalException e) { + return new Result(object, null, e); + } + } + return null; + } + + private static class Result implements VerificationResult { + + private final Throwable throwable; + + private final SignatureVerification verification; + + private final RevObject object; + + public Result(RevObject object, SignatureVerification verification, + Throwable throwable) { + this.object = object; + this.verification = verification; + this.throwable = throwable; + } + + @Override + public Throwable getException() { + return throwable; + } + + @Override + public SignatureVerification getVerification() { + return verification; + } + + @Override + public RevObject getObject() { + return object; + } + + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java index 7e39361ef..81b7bd84c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java @@ -1,42 +1,17 @@ /* - * Copyright (C) 2010,Mathias Kinzler and - * other copyright owners as documented in the project's IP log. + * Copyright (C) 2010, 2020 Mathias Kinzler and others * * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v1.0 which accompanies this - * distribution, is reproduced below, and is available at - * http://www.eclipse.org/org/documents/edl-v10.php + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.api.errors; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.lib.RefUpdate; + /** * Thrown when trying to create a {@link org.eclipse.jgit.lib.Ref} with the same * name as an existing one @@ -44,13 +19,43 @@ public class RefAlreadyExistsException extends GitAPIException { private static final long serialVersionUID = 1L; + private final RefUpdate.Result updateResult; + /** - * Constructor for RefAlreadyExistsException + * Creates a new instance with the given message. * * @param message * error message */ public RefAlreadyExistsException(String message) { + this(message, null); + } + + /** + * Constructor for RefAlreadyExistsException + * + * @param message + * error message + * @param updateResult + * that caused the exception; may be {@code null} + * @since 5.11 + */ + public RefAlreadyExistsException(String message, + @Nullable RefUpdate.Result updateResult) { super(message); + this.updateResult = updateResult; + } + + /** + * Retrieves the {@link org.eclipse.jgit.lib.RefUpdate.Result + * RefUpdate.Result} that caused the exception. + * + * @return the {@link org.eclipse.jgit.lib.RefUpdate.Result + * RefUpdate.Result} or {@code null} if unknown + * @since 5.11 + */ + @Nullable + public RefUpdate.Result getUpdateResult() { + return updateResult; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/WrongObjectTypeException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/WrongObjectTypeException.java new file mode 100644 index 000000000..f639c2f83 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/WrongObjectTypeException.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; + +/** + * A given object is not of an expected object type. + * + * @since 5.11 + */ +public class WrongObjectTypeException extends GitAPIException { + + private static final long serialVersionUID = 1L; + + private String name; + + private int type; + + /** + * Construct a {@link WrongObjectTypeException} for the specified object id, + * giving the expected type. + * + * @param id + * {@link ObjectId} of the object with the unexpected type + * @param type + * expected object type code; see + * {@link Constants}{@code .OBJ_*}. + */ + public WrongObjectTypeException(ObjectId id, int type) { + super(MessageFormat.format(JGitText.get().objectIsNotA, id.name(), + Constants.typeString(type))); + this.name = id.name(); + this.type = type; + } + + /** + * Retrieves the name (SHA-1) of the object. + * + * @return the name + */ + public String getObjectId() { + return name; + } + + /** + * Retrieves the expected type code. See {@link Constants}{@code .OBJ_*}. + * + * @return the type code + */ + public int getExpectedType() { + return type; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java index 2698e2303..1c9e9d7f7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -23,7 +24,7 @@ * @since 4.6 */ public class FilterCommandRegistry { - private static ConcurrentHashMap filterCommandRegistry = new ConcurrentHashMap<>(); + private static Map filterCommandRegistry = new ConcurrentHashMap<>(); /** * Register a {@link org.eclipse.jgit.attributes.FilterCommandFactory} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 8c51a7ac2..671475ed4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -946,12 +946,14 @@ else if (m == null) // called before). Ignore the cached deletion and use what we // find in Merge. Potentially updates the file. if (equalIdAndMode(hId, hMode, mId, mMode)) { - if (initialCheckout) + if (initialCheckout || force) { update(name, mId, mMode); - else + } else { keep(name, dce, f); - } else + } + } else { conflict(name, dce, h, m); + } } } else { // Something in Index @@ -1214,8 +1216,12 @@ private void conflict(String path, DirCacheEntry e, AbstractTreeIterator h, Abst private void keep(String path, DirCacheEntry e, WorkingTreeIterator f) throws IOException { - if (e != null && !FileMode.TREE.equals(e.getFileMode())) + if (e == null) { + return; + } + if (!FileMode.TREE.equals(e.getFileMode())) { builder.add(e); + } if (force) { if (f == null || f.isModified(e, true, walk.getObjectReader())) { kept.add(path); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java index c3b1df992..a37b8bee2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java @@ -13,8 +13,7 @@ import java.io.IOException; /** - * Thrown when a PackFile is found not to contain the pack signature defined by - * git. + * Thrown when a Pack is found not to contain the pack signature defined by git. * * @since 4.5 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java index c484984f8..1fd80867b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java @@ -17,7 +17,7 @@ import org.eclipse.jgit.internal.JGitText; /** - * Thrown when a PackFile previously failed and is known to be unusable + * Thrown when a Pack previously failed and is known to be unusable */ public class PackInvalidException extends IOException { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java index ad5664ceb..44b8e0193 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java @@ -13,7 +13,7 @@ import java.io.IOException; /** - * Thrown when a PackFile no longer matches the PackIndex. + * Thrown when a Pack no longer matches the PackIndex. */ public class PackMismatchException extends IOException { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java index 753822995..07aa7564d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java @@ -16,7 +16,7 @@ import org.eclipse.jgit.internal.JGitText; /** - * Thrown when a PackFile uses a pack version not supported by JGit. + * Thrown when a Pack uses a pack version not supported by JGit. * * @since 4.5 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java b/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java index 32c3a1d28..476c37c1c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java @@ -11,15 +11,15 @@ package org.eclipse.jgit.events; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; /** * Manages a thread-safe list of {@link org.eclipse.jgit.events.RepositoryListener}s. */ public class ListenerList { - private final ConcurrentMap, CopyOnWriteArrayList> lists = new ConcurrentHashMap<>(); + private final Map, CopyOnWriteArrayList> lists = new ConcurrentHashMap<>(); /** * Register a {@link org.eclipse.jgit.events.WorkingTreeModifiedListener}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java index 4059b16b3..ce3ad2239 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java @@ -9,12 +9,10 @@ */ package org.eclipse.jgit.hooks; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; +import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.concurrent.Callable; import org.eclipse.jgit.api.errors.AbortedByHookException; @@ -35,21 +33,21 @@ * the return type which is expected from {@link #call()} * @see Git * Hooks on the git-scm official site - * @since 4.0 + * @since 5.11 */ -abstract class GitHook implements Callable { +public abstract class GitHook implements Callable { private final Repository repo; /** * The output stream to be used by the hook. */ - protected final PrintStream outputStream; + private final OutputStream outputStream; /** * The error stream to be used by the hook. */ - protected final PrintStream errorStream; + private final OutputStream errorStream; /** * Constructor for GitHook. @@ -63,7 +61,7 @@ abstract class GitHook implements Callable { * The output stream the hook must use. {@code null} is allowed, * in which case the hook will use {@code System.out}. */ - protected GitHook(Repository repo, PrintStream outputStream) { + protected GitHook(Repository repo, OutputStream outputStream) { this(repo, outputStream, null); } @@ -79,8 +77,8 @@ protected GitHook(Repository repo, PrintStream outputStream) { * The error stream the hook must use. {@code null} is allowed, * in which case the hook will use {@code System.err}. */ - protected GitHook(Repository repo, PrintStream outputStream, - PrintStream errorStream) { + protected GitHook(Repository repo, OutputStream outputStream, + OutputStream errorStream) { this.repo = repo; this.outputStream = outputStream; this.errorStream = errorStream; @@ -137,7 +135,7 @@ protected String getStdinArgs() { * @return The output stream the hook must use. Never {@code null}, * {@code System.out} is returned by default. */ - protected PrintStream getOutputStream() { + protected OutputStream getOutputStream() { return outputStream == null ? System.out : outputStream; } @@ -147,7 +145,7 @@ protected PrintStream getOutputStream() { * @return The error stream the hook must use. Never {@code null}, * {@code System.err} is returned by default. */ - protected PrintStream getErrorStream() { + protected OutputStream getErrorStream() { return errorStream == null ? System.err : errorStream; } @@ -156,33 +154,47 @@ protected PrintStream getErrorStream() { * * @throws org.eclipse.jgit.api.errors.AbortedByHookException * If the underlying hook script exited with non-zero. + * @throws IOException + * if an IO error occurred */ - protected void doRun() throws AbortedByHookException { + protected void doRun() throws AbortedByHookException, IOException { final ByteArrayOutputStream errorByteArray = new ByteArrayOutputStream(); final TeeOutputStream stderrStream = new TeeOutputStream(errorByteArray, getErrorStream()); - PrintStream hookErrRedirect = null; - try { - hookErrRedirect = new PrintStream(stderrStream, false, - UTF_8.name()); - } catch (UnsupportedEncodingException e) { - // UTF-8 is guaranteed to be available - } Repository repository = getRepository(); FS fs = repository.getFS(); if (fs == null) { fs = FS.DETECTED; } ProcessResult result = fs.runHookIfPresent(repository, getHookName(), - getParameters(), getOutputStream(), hookErrRedirect, + getParameters(), getOutputStream(), stderrStream, getStdinArgs()); if (result.isExecutedWithError()) { - throw new AbortedByHookException( - new String(errorByteArray.toByteArray(), UTF_8), - getHookName(), result.getExitCode()); + handleError(new String(errorByteArray.toByteArray(), + Charset.defaultCharset().name()), result); } } + /** + * Process that the hook exited with an error. This default implementation + * throws an {@link AbortedByHookException }. Hooks which need a different + * behavior can overwrite this method. + * + * @param message + * error message + * @param result + * The process result of the hook + * @throws AbortedByHookException + * When the hook should be aborted + * @since 5.11 + */ + protected void handleError(String message, + final ProcessResult result) + throws AbortedByHookException { + throw new AbortedByHookException(message, getHookName(), + result.getExitCode()); + } + /** * Check whether a 'native' (i.e. script) hook is installed in the * repository. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java index 0b61ebea3..b9dafcca3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java @@ -14,6 +14,7 @@ import org.eclipse.jgit.api.errors.AbortedByHookException; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.ProcessResult; /** * The post-commit hook implementation. This hook is run after the @@ -73,4 +74,16 @@ public String getHookName() { return NAME; } + + /** + * Overwrites the default implementation to never throw an + * {@link AbortedByHookException}, as the commit has already been done and + * the exit code of the post-commit hook has no effect. + */ + @Override + protected void handleError(String message, ProcessResult result) + throws AbortedByHookException { + // Do nothing as the exit code of the post-commit hook has no effect. + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java index d7e4f79d2..9dd565ff0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014, Andrey Loskutov and others + * Copyright (C) 2014, 2021 Andrey Loskutov and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -14,8 +14,11 @@ import static org.eclipse.jgit.ignore.internal.Strings.stripTrailing; import static org.eclipse.jgit.ignore.internal.Strings.stripTrailingWhitespace; +import java.text.MessageFormat; + import org.eclipse.jgit.errors.InvalidPatternException; import org.eclipse.jgit.ignore.internal.PathMatcher; +import org.eclipse.jgit.internal.JGitText; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,11 +39,11 @@ public class FastIgnoreRule { */ public static final char PATH_SEPARATOR = '/'; - private final IMatcher matcher; + private IMatcher matcher; - private final boolean inverse; + private boolean inverse; - private final boolean dirOnly; + private boolean dirOnly; /** * Constructor for FastIgnoreRule @@ -52,8 +55,23 @@ public class FastIgnoreRule { * (comment), this rule doesn't match anything. */ public FastIgnoreRule(String pattern) { - if (pattern == null) + this(); + try { + parse(pattern); + } catch (InvalidPatternException e) { + LOG.error(MessageFormat.format(JGitText.get().badIgnorePattern, + e.getPattern()), e); + } + } + + FastIgnoreRule() { + matcher = IMatcher.NO_MATCH; + } + + void parse(String pattern) throws InvalidPatternException { + if (pattern == null) { throw new IllegalArgumentException("Pattern must not be null!"); //$NON-NLS-1$ + } if (pattern.length() == 0) { dirOnly = false; inverse = false; @@ -90,15 +108,8 @@ public FastIgnoreRule(String pattern) { return; } } - IMatcher m; - try { - m = PathMatcher.createPathMatcher(pattern, - Character.valueOf(PATH_SEPARATOR), dirOnly); - } catch (InvalidPatternException e) { - m = NO_MATCH; - LOG.error(e.getMessage(), e); - } - this.matcher = m; + this.matcher = PathMatcher.createPathMatcher(pattern, + Character.valueOf(PATH_SEPARATOR), dirOnly); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java index 0bc612491..4e7f126a6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Red Hat Inc. and others + * Copyright (C) 2010, 2021 Red Hat Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -15,16 +15,26 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.eclipse.jgit.internal.JGitText; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Represents a bundle of ignore rules inherited from a base directory. * * This class is not thread safe, it maintains state about the last match. */ public class IgnoreNode { + + private static final Logger LOG = LoggerFactory.getLogger(IgnoreNode.class); + /** Result from {@link IgnoreNode#isIgnored(String, boolean)}. */ public enum MatchResult { /** The file is not ignored, due to a rule saying its not ignored. */ @@ -52,7 +62,7 @@ public enum MatchResult { * Create an empty ignore node with no rules. */ public IgnoreNode() { - rules = new ArrayList<>(); + this(new ArrayList<>()); } /** @@ -75,15 +85,47 @@ public IgnoreNode(List rules) { * Error thrown when reading an ignore file. */ public void parse(InputStream in) throws IOException { + parse(null, in); + } + + /** + * Parse files according to gitignore standards. + * + * @param sourceName + * identifying the source of the stream + * @param in + * input stream holding the standard ignore format. The caller is + * responsible for closing the stream. + * @throws java.io.IOException + * Error thrown when reading an ignore file. + * @since 5.11 + */ + public void parse(String sourceName, InputStream in) throws IOException { BufferedReader br = asReader(in); String txt; + int lineNumber = 1; while ((txt = br.readLine()) != null) { if (txt.length() > 0 && !txt.startsWith("#") && !txt.equals("/")) { //$NON-NLS-1$ //$NON-NLS-2$ - FastIgnoreRule rule = new FastIgnoreRule(txt); + FastIgnoreRule rule = new FastIgnoreRule(); + try { + rule.parse(txt); + } catch (InvalidPatternException e) { + if (sourceName != null) { + LOG.error(MessageFormat.format( + JGitText.get().badIgnorePatternFull, sourceName, + Integer.toString(lineNumber), e.getPattern(), + e.getLocalizedMessage()), e); + } else { + LOG.error(MessageFormat.format( + JGitText.get().badIgnorePattern, + e.getPattern()), e); + } + } if (!rule.isEmpty()) { rules.add(rule); } } + lineNumber++; } } @@ -135,7 +177,8 @@ public MatchResult isIgnored(String entryPath, boolean isDirectory) { * undetermined * @since 4.11 */ - public Boolean checkIgnored(String entryPath, boolean isDirectory) { + public @Nullable Boolean checkIgnored(String entryPath, + boolean isDirectory) { // Parse rules in the reverse order that they were read because later // rules have higher priority for (int i = rules.size() - 1; i > -1; i--) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 892657d5d..9d215ca45 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, 2013 Sasa Zivkov - * Copyright (C) 2012, Research In Motion Limited and others + * Copyright (C) 2012, 2021 Research In Motion Limited and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -58,6 +58,8 @@ public static JGitText get() { /***/ public String badEntryName; /***/ public String badEscape; /***/ public String badGroupHeader; + /***/ public String badIgnorePattern; + /***/ public String badIgnorePatternFull; /***/ public String badObjectType; /***/ public String badRef; /***/ public String badSectionEntry; @@ -261,6 +263,7 @@ public static JGitText get() { /***/ public String downloadCancelledDuringIndexing; /***/ public String duplicateAdvertisementsOf; /***/ public String duplicateRef; + /***/ public String duplicateRefAttribute; /***/ public String duplicateRemoteRefUpdateIsIllegal; /***/ public String duplicateStagesNotAllowed; /***/ public String eitherGitDirOrWorkTreeRequired; @@ -338,6 +341,10 @@ public static JGitText get() { /***/ public String hoursAgo; /***/ public String httpConfigCannotNormalizeURL; /***/ public String httpConfigInvalidURL; + /***/ public String httpFactoryInUse; + /***/ public String httpPreAuthTooLate; + /***/ public String httpUserInfoDecodeError; + /***/ public String httpWrongConnectionType; /***/ public String hugeIndexesAreNotSupportedByJgitYet; /***/ public String hunkBelongsToAnotherFile; /***/ public String hunkDisconnectedFromFile; @@ -591,6 +598,7 @@ public static JGitText get() { /***/ public String reftableDirExists; /***/ public String reftableRecordsMustIncrease; /***/ public String refUpdateReturnCodeWas; + /***/ public String remoteBranchNotFound; /***/ public String remoteConfigHasNoURIAssociated; /***/ public String remoteDoesNotHaveSpec; /***/ public String remoteDoesNotSupportSmartHTTPPush; @@ -645,7 +653,9 @@ public static JGitText get() { /***/ public String shortReadOfBlock; /***/ public String shortReadOfOptionalDIRCExtensionExpectedAnotherBytes; /***/ public String shortSkipOfBlock; - /***/ public String signingNotSupportedOnTag; + /***/ public String signatureVerificationError; + /***/ public String signatureVerificationUnavailable; + /***/ public String signedTagMessageNoLf; /***/ public String signingServiceUnavailable; /***/ public String similarityScoreMustBeWithinBounds; /***/ public String skipMustBeNonNegative; @@ -786,6 +796,13 @@ public static JGitText get() { /***/ public String URINotSupported; /***/ public String userConfigInvalid; /***/ public String validatingGitModules; + /***/ public String verifySignatureBad; + /***/ public String verifySignatureExpired; + /***/ public String verifySignatureGood; + /***/ public String verifySignatureIssuer; + /***/ public String verifySignatureKey; + /***/ public String verifySignatureMade; + /***/ public String verifySignatureTrust; /***/ public String walkFailure; /***/ public String wantNoSpaceWithCapabilities; /***/ public String wantNotValid; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java deleted file mode 100644 index 5ddbcbd0e..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.KetchConstants.TERM; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeoutException; - -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.TreeFormatter; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.time.ProposedTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The initial {@link Round} for a leaderless repository, used to establish a - * leader. - */ -class ElectionRound extends Round { - private static final Logger log = LoggerFactory.getLogger(ElectionRound.class); - - private long term; - - ElectionRound(KetchLeader leader, LogIndex head) { - super(leader, head); - } - - @Override - void start() throws IOException { - ObjectId id; - try (Repository git = leader.openRepository(); - ProposedTimestamp ts = getSystem().getClock().propose(); - ObjectInserter inserter = git.newObjectInserter()) { - id = bumpTerm(git, ts, inserter); - inserter.flush(); - blockUntil(ts); - } - runAsync(id); - } - - @Override - void success() { - // Do nothing upon election, KetchLeader will copy the term. - } - - long getTerm() { - return term; - } - - private ObjectId bumpTerm(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException { - CommitBuilder b = new CommitBuilder(); - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(acceptedOldIndex); - if (getSystem().requireMonotonicLeaderElections()) { - if (ts.read(SECONDS) < c.getCommitTime()) { - throw new TimeIsUncertainException(); - } - } - b.setTreeId(c.getTree()); - b.setParentId(acceptedOldIndex); - term = parseTerm(c.getFooterLines(TERM)) + 1; - } - } else { - term = 1; - b.setTreeId(inserter.insert(new TreeFormatter())); - } - - StringBuilder msg = new StringBuilder(); - msg.append(KetchConstants.TERM.getName()) - .append(": ") //$NON-NLS-1$ - .append(term); - - String tag = leader.getSystem().newLeaderTag(); - if (tag != null && !tag.isEmpty()) { - msg.append(' ').append(tag); - } - - b.setAuthor(leader.getSystem().newCommitter(ts)); - b.setCommitter(b.getAuthor()); - b.setMessage(msg.toString()); - - if (log.isDebugEnabled()) { - log.debug("Trying to elect myself " + b.getMessage()); //$NON-NLS-1$ - } - return inserter.insert(b); - } - - private static long parseTerm(List footer) { - if (footer.isEmpty()) { - return 0; - } - - String s = footer.get(0); - int p = s.indexOf(' '); - if (p > 0) { - s = s.substring(0, p); - } - return Long.parseLong(s, 10); - } - - private void blockUntil(ProposedTimestamp ts) throws IOException { - try { - ts.blockUntil(getSystem().getMaxWaitForMonotonicClock()); - } catch (InterruptedException | TimeoutException e) { - throw new TimeIsUncertainException(e); - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java deleted file mode 100644 index f4a7f592f..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.revwalk.FooterKey; - -/** - * Frequently used constants in a Ketch system. - */ -public class KetchConstants { - /** - * Default reference namespace holding {@link #ACCEPTED} and - * {@link #COMMITTED} references and the {@link #STAGE} sub-namespace. - */ - public static final String DEFAULT_TXN_NAMESPACE = "refs/txn/"; //$NON-NLS-1$ - - /** Reference name holding the RefTree accepted by a follower. */ - public static final String ACCEPTED = "accepted"; //$NON-NLS-1$ - - /** Reference name holding the RefTree known to be committed. */ - public static final String COMMITTED = "committed"; //$NON-NLS-1$ - - /** Reference subdirectory holding proposed heads. */ - public static final String STAGE = "stage/"; //$NON-NLS-1$ - - /** Footer containing the current term. */ - public static final FooterKey TERM = new FooterKey("Term"); //$NON-NLS-1$ - - /** Section for Ketch configuration ({@code ketch}). */ - public static final String CONFIG_SECTION_KETCH = "ketch"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-type}) */ - public static final String CONFIG_KEY_TYPE = "ketch-type"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-commit}) */ - public static final String CONFIG_KEY_COMMIT = "ketch-commit"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-speed}) */ - public static final String CONFIG_KEY_SPEED = "ketch-speed"; //$NON-NLS-1$ - - private KetchConstants() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java deleted file mode 100644 index 743d1939c..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java +++ /dev/null @@ -1,604 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.CANDIDATE; -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.LEADER; -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.SHUTDOWN; -import static org.eclipse.jgit.internal.ketch.KetchReplica.Participation.FOLLOWER_ONLY; -import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A leader managing consensus across remote followers. - *

    - * A leader instance starts up in - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#CANDIDATE} and tries - * to begin a new term by sending an - * {@link org.eclipse.jgit.internal.ketch.ElectionRound} to all replicas. Its - * term starts if a majority of replicas have accepted this leader instance for - * the term. - *

    - * Once elected by a majority the instance enters - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER} and runs - * proposals offered to {@link #queueProposal(Proposal)}. This continues until - * the leader is timed out for inactivity, or is deposed by a competing leader - * gaining its own majority. - *

    - * Once timed out or deposed this {@code KetchLeader} instance should be - * discarded, and a new instance takes over. - *

    - * Each leader instance coordinates a group of - * {@link org.eclipse.jgit.internal.ketch.KetchReplica}s. Replica instances are - * owned by the leader instance and must be discarded when the leader is - * discarded. - *

    - * In Ketch all push requests are issued through the leader. The steps are as - * follows (see {@link org.eclipse.jgit.internal.ketch.KetchPreReceive} for an - * example): - *

      - *
    • Create a {@link org.eclipse.jgit.internal.ketch.Proposal} with the - * {@link org.eclipse.jgit.transport.ReceiveCommand}s that represent the push. - *
    • Invoke {@link #queueProposal(Proposal)} on the leader instance. - *
    • Wait for consensus with - * {@link org.eclipse.jgit.internal.ketch.Proposal#await()}. - *
    • To examine the status of the push, check - * {@link org.eclipse.jgit.internal.ketch.Proposal#getCommands()}, looking at - * {@link org.eclipse.jgit.internal.storage.reftree.Command#getResult()}. - *
    - *

    - * The leader gains consensus by first pushing the needed objects and a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree} representing the - * desired target repository state to the {@code refs/txn/accepted} branch on - * each of the replicas. Once a majority has succeeded, the leader commits the - * state by either pushing the {@code refs/txn/accepted} value to - * {@code refs/txn/committed} (for Ketch-aware replicas) or by pushing updates - * to {@code refs/heads/master}, etc. for stock Git replicas. - *

    - * Internally, the actual transport to replicas is performed on background - * threads via the {@link org.eclipse.jgit.internal.ketch.KetchSystem}'s - * executor service. For performance, the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}, - * {@link org.eclipse.jgit.internal.ketch.KetchReplica} and - * {@link org.eclipse.jgit.internal.ketch.Proposal} objects share some state, - * and may invoke each other's methods on different threads. This access is - * protected by the leader's {@link #lock} object. Care must be taken to prevent - * concurrent access by correctly obtaining the leader's lock. - */ -public abstract class KetchLeader { - private static final Logger log = LoggerFactory.getLogger(KetchLeader.class); - - /** Current state of the leader instance. */ - public enum State { - /** Newly created instance trying to elect itself leader. */ - CANDIDATE, - - /** Leader instance elected by a majority. */ - LEADER, - - /** Instance has been deposed by another with a more recent term. */ - DEPOSED, - - /** Leader has been gracefully shutdown, e.g. due to inactivity. */ - SHUTDOWN; - } - - private final KetchSystem system; - - /** Leader's knowledge of replicas for this repository. */ - private KetchReplica[] voters; - private KetchReplica[] followers; - private LocalReplica self; - - /** - * Lock protecting all data within this leader instance. - *

    - * This lock extends into the {@link KetchReplica} instances used by the - * leader. They share the same lock instance to simplify concurrency. - */ - final Lock lock; - - private State state = CANDIDATE; - - /** Term of this leader, once elected. */ - private long term; - - /** - * Pending proposals accepted into the queue in FIFO order. - *

    - * These proposals were preflighted and do not contain any conflicts with - * each other and their expectations matched the leader's local view of the - * agreed upon {@code refs/txn/accepted} tree. - */ - private final List queued; - - /** - * State of the repository's RefTree after applying all entries in - * {@link #queued}. New proposals must be consistent with this tree to be - * appended to the end of {@link #queued}. - *

    - * Must be deep-copied with {@link RefTree#copy()} if - * {@link #roundHoldsReferenceToRefTree} is {@code true}. - */ - private RefTree refTree; - - /** - * If {@code true} {@link #refTree} must be duplicated before queuing the - * next proposal. The {@link #refTree} was passed into the constructor of a - * {@link ProposalRound}, and that external reference to the {@link RefTree} - * object is held by the proposal until it materializes the tree object in - * the object store. This field is set {@code true} when the proposal begins - * execution and set {@code false} once tree objects are persisted in the - * local repository's object store or {@link #refTree} is replaced with a - * copy to isolate it from any running rounds. - *

    - * If proposals arrive less frequently than the {@code RefTree} is written - * out to the repository the {@link #roundHoldsReferenceToRefTree} behavior - * avoids duplicating {@link #refTree}, reducing both time and memory used. - * However if proposals arrive more frequently {@link #refTree} must be - * duplicated to prevent newly queued proposals from corrupting the - * {@link #runningRound}. - */ - volatile boolean roundHoldsReferenceToRefTree; - - /** End of the leader's log. */ - private LogIndex headIndex; - - /** Leader knows this (and all prior) states are committed. */ - private LogIndex committedIndex; - - /** - * Is the leader idle with no work pending? If {@code true} there is no work - * for the leader (normal state). This field is {@code false} when the - * leader thread is scheduled for execution, or while {@link #runningRound} - * defines a round in progress. - */ - private boolean idle; - - /** Current round the leader is preparing and waiting for a vote on. */ - private Round runningRound; - - /** - * Construct a leader for a Ketch instance. - * - * @param system - * Ketch system configuration the leader must adhere to. - */ - protected KetchLeader(KetchSystem system) { - this.system = system; - this.lock = new ReentrantLock(true /* fair */); - this.queued = new ArrayList<>(4); - this.idle = true; - } - - /** @return system configuration. */ - KetchSystem getSystem() { - return system; - } - - /** - * Configure the replicas used by this Ketch instance. - *

    - * Replicas should be configured once at creation before any proposals are - * executed. Once elections happen, reconfiguration is a complicated - * concept that is not currently supported. - * - * @param replicas - * members participating with the same repository. - */ - public void setReplicas(Collection replicas) { - List v = new ArrayList<>(5); - List f = new ArrayList<>(5); - for (KetchReplica r : replicas) { - switch (r.getParticipation()) { - case FULL: - v.add(r); - break; - - case FOLLOWER_ONLY: - f.add(r); - break; - } - } - - Collection validVoters = validVoterCounts(); - if (!validVoters.contains(Integer.valueOf(v.size()))) { - throw new IllegalArgumentException(MessageFormat.format( - KetchText.get().unsupportedVoterCount, - Integer.valueOf(v.size()), - validVoters)); - } - - LocalReplica me = findLocal(v); - if (me == null) { - throw new IllegalArgumentException( - KetchText.get().localReplicaRequired); - } - - lock.lock(); - try { - voters = v.toArray(new KetchReplica[0]); - followers = f.toArray(new KetchReplica[0]); - self = me; - } finally { - lock.unlock(); - } - } - - private static Collection validVoterCounts() { - @SuppressWarnings("boxing") - Integer[] valid = { - // An odd number of voting replicas is required. - 1, 3, 5, 7, 9 }; - return Arrays.asList(valid); - } - - private static LocalReplica findLocal(Collection voters) { - for (KetchReplica r : voters) { - if (r instanceof LocalReplica) { - return (LocalReplica) r; - } - } - return null; - } - - /** - * Get an instance of the repository for use by a leader thread. - *

    - * The caller will close the repository. - * - * @return opened repository for use by the leader thread. - * @throws java.io.IOException - * cannot reopen the repository for the leader. - */ - protected abstract Repository openRepository() throws IOException; - - /** - * Queue a reference update proposal for consensus. - *

    - * This method does not wait for consensus to be reached. The proposal is - * checked to look for risks of conflicts, and then submitted into the queue - * for distribution as soon as possible. - *

    - * Callers must use {@link org.eclipse.jgit.internal.ketch.Proposal#await()} - * to see if the proposal is done. - * - * @param proposal - * the proposed reference updates to queue for consideration. - * Once execution is complete the individual reference result - * fields will be populated with the outcome. - * @throws java.lang.InterruptedException - * current thread was interrupted. The proposal may have been - * aborted if it was not yet queued for execution. - * @throws java.io.IOException - * unrecoverable error preventing proposals from being attempted - * by this leader. - */ - public void queueProposal(Proposal proposal) - throws InterruptedException, IOException { - try { - lock.lockInterruptibly(); - } catch (InterruptedException e) { - proposal.abort(); - throw e; - } - try { - if (refTree == null) { - initialize(); - for (Proposal p : queued) { - refTree.apply(p.getCommands()); - } - } else if (roundHoldsReferenceToRefTree) { - refTree = refTree.copy(); - roundHoldsReferenceToRefTree = false; - } - - if (!refTree.apply(proposal.getCommands())) { - // A conflict exists so abort the proposal. - proposal.abort(); - return; - } - - queued.add(proposal); - proposal.notifyState(QUEUED); - - if (idle) { - scheduleLeader(); - } - } finally { - lock.unlock(); - } - } - - private void initialize() throws IOException { - try (Repository git = openRepository(); RevWalk rw = new RevWalk(git)) { - self.initialize(git); - - ObjectId accepted = self.getTxnAccepted(); - if (!ObjectId.zeroId().equals(accepted)) { - RevCommit c = rw.parseCommit(accepted); - headIndex = LogIndex.unknown(accepted); - refTree = RefTree.read(rw.getObjectReader(), c.getTree()); - } else { - headIndex = LogIndex.unknown(ObjectId.zeroId()); - refTree = RefTree.newEmptyTree(); - } - } - } - - private void scheduleLeader() { - idle = false; - system.getExecutor().execute(this::runLeader); - } - - private void runLeader() { - Round round; - lock.lock(); - try { - switch (state) { - case CANDIDATE: - round = new ElectionRound(this, headIndex); - break; - - case LEADER: - round = newProposalRound(); - break; - - case DEPOSED: - case SHUTDOWN: - default: - log.warn("Leader cannot run {}", state); //$NON-NLS-1$ - // TODO(sop): Redirect proposals. - return; - } - } finally { - lock.unlock(); - } - - try { - round.start(); - } catch (IOException e) { - // TODO(sop) Depose leader if it cannot use its repository. - log.error(KetchText.get().leaderFailedToStore, e); - lock.lock(); - try { - nextRound(); - } finally { - lock.unlock(); - } - } - } - - private ProposalRound newProposalRound() { - List todo = new ArrayList<>(queued); - queued.clear(); - roundHoldsReferenceToRefTree = true; - return new ProposalRound(this, headIndex, todo, refTree); - } - - /** @return term of this leader's reign. */ - long getTerm() { - return term; - } - - /** @return end of the leader's log. */ - LogIndex getHead() { - return headIndex; - } - - /** - * @return state leader knows it has committed across a quorum of replicas. - */ - LogIndex getCommitted() { - return committedIndex; - } - - boolean isIdle() { - return idle; - } - - void runAsync(Round round) { - lock.lock(); - try { - // End of the log is this round. Once transport begins it is - // reasonable to assume at least one replica will eventually get - // this, and there is reasonable probability it commits. - headIndex = round.acceptedNewIndex; - runningRound = round; - - for (KetchReplica replica : voters) { - replica.pushTxnAcceptedAsync(round); - } - for (KetchReplica replica : followers) { - replica.pushTxnAcceptedAsync(round); - } - } finally { - lock.unlock(); - } - } - - /** - * Asynchronous signal from a replica after completion. - *

    - * Must be called while {@link #lock} is held by the replica. - * - * @param replica - * replica posting a completion event. - */ - void onReplicaUpdate(KetchReplica replica) { - if (log.isDebugEnabled()) { - log.debug("Replica {} finished:\n{}", //$NON-NLS-1$ - replica.describeForLog(), snapshot()); - } - - if (replica.getParticipation() == FOLLOWER_ONLY) { - // Followers cannot vote, so votes haven't changed. - return; - } else if (runningRound == null) { - // No round running, no need to tally votes. - return; - } - - assert headIndex.equals(runningRound.acceptedNewIndex); - int matching = 0; - for (KetchReplica r : voters) { - if (r.hasAccepted(headIndex)) { - matching++; - } - } - - int quorum = voters.length / 2 + 1; - boolean success = matching >= quorum; - if (!success) { - return; - } - - switch (state) { - case CANDIDATE: - term = ((ElectionRound) runningRound).getTerm(); - state = LEADER; - if (log.isDebugEnabled()) { - log.debug("Won election, running term " + term); //$NON-NLS-1$ - } - - //$FALL-THROUGH$ - case LEADER: - committedIndex = headIndex; - if (log.isDebugEnabled()) { - log.debug("Committed {} in term {}", //$NON-NLS-1$ - committedIndex.describeForLog(), - Long.valueOf(term)); - } - nextRound(); - commitAsync(replica); - notifySuccess(runningRound); - if (log.isDebugEnabled()) { - log.debug("Leader state:\n{}", snapshot()); //$NON-NLS-1$ - } - break; - - default: - log.debug("Leader ignoring replica while in {}", state); //$NON-NLS-1$ - break; - } - } - - private void notifySuccess(Round round) { - // Drop the leader lock while notifying Proposal listeners. - lock.unlock(); - try { - round.success(); - } finally { - lock.lock(); - } - } - - private void commitAsync(KetchReplica caller) { - for (KetchReplica r : voters) { - if (r == caller) { - continue; - } - if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { - r.pushCommitAsync(committedIndex); - } - } - for (KetchReplica r : followers) { - if (r == caller) { - continue; - } - if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { - r.pushCommitAsync(committedIndex); - } - } - } - - /** Schedule the next round; invoked while {@link #lock} is held. */ - void nextRound() { - runningRound = null; - - if (queued.isEmpty()) { - idle = true; - } else { - // Caller holds lock. Reschedule leader on a new thread so - // the call stack can unwind and lock is not held unexpectedly - // during prepare for the next round. - scheduleLeader(); - } - } - - /** - * Snapshot this leader - * - * @return snapshot of this leader - */ - public LeaderSnapshot snapshot() { - lock.lock(); - try { - LeaderSnapshot s = new LeaderSnapshot(); - s.state = state; - s.term = term; - s.headIndex = headIndex; - s.committedIndex = committedIndex; - s.idle = isIdle(); - for (KetchReplica r : voters) { - s.replicas.add(r.snapshot()); - } - for (KetchReplica r : followers) { - s.replicas.add(r.snapshot()); - } - return s; - } finally { - lock.unlock(); - } - } - - /** - * Gracefully shutdown this leader and cancel outstanding operations. - */ - public void shutdown() { - lock.lock(); - try { - if (state != SHUTDOWN) { - state = SHUTDOWN; - for (KetchReplica r : voters) { - r.shutdown(); - } - for (KetchReplica r : followers) { - r.shutdown(); - } - } - } finally { - lock.unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public String toString() { - return snapshot().toString(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java deleted file mode 100644 index e01fb3ae5..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.net.URISyntaxException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.eclipse.jgit.internal.storage.dfs.DfsRepository; -import org.eclipse.jgit.lib.Repository; - -/** - * A cache of live leader instances, keyed by repository. - *

    - * Ketch only assigns a leader to a repository when needed. If - * {@link #get(Repository)} is called for a repository that does not have a - * leader, the leader is created and added to the cache. - */ -public class KetchLeaderCache { - private final KetchSystem system; - private final ConcurrentMap leaders; - private final Lock startLock; - - /** - * Initialize a new leader cache. - * - * @param system - * system configuration for the leaders - */ - public KetchLeaderCache(KetchSystem system) { - this.system = system; - leaders = new ConcurrentHashMap<>(); - startLock = new ReentrantLock(true /* fair */); - } - - /** - * Lookup the leader instance for a given repository. - * - * @param repo - * repository to get the leader for. - * @return the leader instance for the repository. - * @throws java.net.URISyntaxException - * remote configuration contains an invalid URL. - */ - public KetchLeader get(Repository repo) - throws URISyntaxException { - String key = computeKey(repo); - KetchLeader leader = leaders.get(key); - if (leader != null) { - return leader; - } - return startLeader(key, repo); - } - - private KetchLeader startLeader(String key, Repository repo) - throws URISyntaxException { - startLock.lock(); - try { - KetchLeader leader = leaders.get(key); - if (leader != null) { - return leader; - } - leader = system.createLeader(repo); - leaders.put(key, leader); - return leader; - } finally { - startLock.unlock(); - } - } - - private static String computeKey(Repository repo) { - if (repo instanceof DfsRepository) { - DfsRepository dfs = (DfsRepository) repo; - return dfs.getDescription().getRepositoryName(); - } - - if (repo.getDirectory() != null) { - return repo.getDirectory().toURI().toString(); - } - - throw new IllegalArgumentException(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java deleted file mode 100644 index 1c9535f7b..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.Collection; - -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.transport.PreReceiveHook; -import org.eclipse.jgit.transport.ProgressSpinner; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.ReceivePack; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * PreReceiveHook for handling push traffic in a Ketch system. - *

    - * Install an instance on {@link org.eclipse.jgit.transport.ReceivePack} to - * capture the commands and other connection state and relay them through the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}, allowing the leader to - * gain consensus about the new reference state. - */ -public class KetchPreReceive implements PreReceiveHook { - private static final Logger log = LoggerFactory.getLogger(KetchPreReceive.class); - - private final KetchLeader leader; - - /** - * Construct a hook executing updates through a - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. - * - * @param leader - * leader for this repository. - */ - public KetchPreReceive(KetchLeader leader) { - this.leader = leader; - } - - /** {@inheritDoc} */ - @Override - public void onPreReceive(ReceivePack rp, Collection cmds) { - cmds = ReceiveCommand.filter(cmds, NOT_ATTEMPTED); - if (cmds.isEmpty()) { - return; - } - - try { - Proposal proposal = new Proposal(rp.getRevWalk(), cmds) - .setPushCertificate(rp.getPushCertificate()) - .setAuthor(rp.getRefLogIdent()) - .setMessage("push"); //$NON-NLS-1$ - leader.queueProposal(proposal); - if (proposal.isDone()) { - // This failed fast, e.g. conflict or bad precondition. - return; - } - - ProgressSpinner spinner = new ProgressSpinner( - rp.getMessageOutputStream()); - if (proposal.getState() == QUEUED) { - waitForQueue(proposal, spinner); - } - if (!proposal.isDone()) { - waitForPropose(proposal, spinner); - } - } catch (IOException | InterruptedException e) { - String msg = JGitText.get().transactionAborted; - for (ReceiveCommand cmd : cmds) { - if (cmd.getResult() == NOT_ATTEMPTED) { - cmd.setResult(REJECTED_OTHER_REASON, msg); - } - } - log.error(msg, e); - } - } - - private void waitForQueue(Proposal proposal, ProgressSpinner spinner) - throws InterruptedException { - spinner.beginTask(KetchText.get().waitingForQueue, 1, SECONDS); - while (!proposal.awaitStateChange(QUEUED, 250, MILLISECONDS)) { - spinner.update(); - } - switch (proposal.getState()) { - case RUNNING: - default: - spinner.endTask(KetchText.get().starting); - break; - - case EXECUTED: - spinner.endTask(KetchText.get().accepted); - break; - - case ABORTED: - spinner.endTask(KetchText.get().failed); - break; - } - } - - private void waitForPropose(Proposal proposal, ProgressSpinner spinner) - throws InterruptedException { - spinner.beginTask(KetchText.get().proposingUpdates, 2, SECONDS); - while (!proposal.await(250, MILLISECONDS)) { - spinner.update(); - } - spinner.endTask(proposal.getState() == EXECUTED - ? KetchText.get().accepted - : KetchText.get().failed); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java deleted file mode 100644 index a9a694a56..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java +++ /dev/null @@ -1,758 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.BATCHED; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.FAST; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.CURRENT; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.CREATE; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.SystemReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A Ketch replica, either {@link org.eclipse.jgit.internal.ketch.LocalReplica} - * or {@link org.eclipse.jgit.internal.ketch.RemoteGitReplica}. - *

    - * Replicas can be either a stock Git replica, or a Ketch-aware replica. - *

    - * A stock Git replica has no special knowledge of Ketch and simply stores - * objects and references. Ketch communicates with the stock Git replica using - * the Git push wire protocol. The - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} commits an agreed upon - * state by pushing all references to the Git replica, for example - * {@code "refs/heads/master"} is pushed during commit. Stock Git replicas use - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS} to - * record the final state. - *

    - * Ketch-aware replicas understand the {@code RefTree} sent during the proposal - * and during commit are able to update their own reference space to match the - * state represented by the {@code RefTree}. Ketch-aware replicas typically use - * a {@link org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase} and - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#TXN_COMMITTED} - * to record the final state. - *

    - * KetchReplica instances are tightly coupled with a single - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. Some state may be - * accessed by the leader thread and uses the leader's own - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} to protect shared - * data. - */ -public abstract class KetchReplica { - static final Logger log = LoggerFactory.getLogger(KetchReplica.class); - private static final byte[] PEEL = { ' ', '^' }; - - /** Participation of a replica in establishing consensus. */ - public enum Participation { - /** Replica can vote. */ - FULL, - - /** Replica does not vote, but tracks leader. */ - FOLLOWER_ONLY; - } - - /** How this replica wants to receive Ketch commit operations. */ - public enum CommitMethod { - /** All references are pushed to the peer as standard Git. */ - ALL_REFS, - - /** Only {@code refs/txn/committed} is written/updated. */ - TXN_COMMITTED; - } - - /** Delay before committing to a replica. */ - public enum CommitSpeed { - /** - * Send the commit immediately, even if it could be batched with the - * next proposal. - */ - FAST, - - /** - * If the next proposal is available, batch the commit with it, - * otherwise just send the commit. This generates less network use, but - * may provide slower consistency on the replica. - */ - BATCHED; - } - - /** Current state of a replica. */ - public enum State { - /** Leader has not yet contacted the replica. */ - UNKNOWN, - - /** Replica is behind the consensus. */ - LAGGING, - - /** Replica matches the consensus. */ - CURRENT, - - /** Replica has a different (or unknown) history. */ - DIVERGENT, - - /** Replica's history contains the leader's history. */ - AHEAD, - - /** Replica can not be contacted. */ - OFFLINE; - } - - private final KetchLeader leader; - private final String replicaName; - private final Participation participation; - private final CommitMethod commitMethod; - private final CommitSpeed commitSpeed; - private final long minRetryMillis; - private final long maxRetryMillis; - private final Map> staged; - private final Map running; - private final Map waiting; - private final List queued; - - /** - * Value known for {@code "refs/txn/accepted"}. - *

    - * Raft literature refers to this as {@code matchIndex}. - */ - private ObjectId txnAccepted; - - /** - * Value known for {@code "refs/txn/committed"}. - *

    - * Raft literature refers to this as {@code commitIndex}. In traditional - * Raft this is a state variable inside the follower implementation, but - * Ketch keeps it in the leader. - */ - private ObjectId txnCommitted; - - /** What is happening with this replica. */ - private State state = UNKNOWN; - private String error; - - /** Scheduled retry due to communication failure. */ - private Future retryFuture; - private long lastRetryMillis; - private long retryAtMillis; - - /** - * Configure a replica representation. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this replica for debugging. - * @param cfg - * how Ketch should treat the replica. - */ - protected KetchReplica(KetchLeader leader, String name, ReplicaConfig cfg) { - this.leader = leader; - this.replicaName = name; - this.participation = cfg.getParticipation(); - this.commitMethod = cfg.getCommitMethod(); - this.commitSpeed = cfg.getCommitSpeed(); - this.minRetryMillis = cfg.getMinRetry(MILLISECONDS); - this.maxRetryMillis = cfg.getMaxRetry(MILLISECONDS); - this.staged = new HashMap<>(); - this.running = new HashMap<>(); - this.waiting = new HashMap<>(); - this.queued = new ArrayList<>(4); - } - - /** - * Get system configuration. - * - * @return system configuration. - */ - public KetchSystem getSystem() { - return getLeader().getSystem(); - } - - /** - * Get leader instance this replica follows. - * - * @return leader instance this replica follows. - */ - public KetchLeader getLeader() { - return leader; - } - - /** - * Get unique-ish name for debugging. - * - * @return unique-ish name for debugging. - */ - public String getName() { - return replicaName; - } - - /** - * Get description of this replica for error/debug logging purposes. - * - * @return description of this replica for error/debug logging purposes. - */ - protected String describeForLog() { - return getName(); - } - - /** - * Get how the replica participates in this Ketch system. - * - * @return how the replica participates in this Ketch system. - */ - public Participation getParticipation() { - return participation; - } - - /** - * Get how Ketch will commit to the repository. - * - * @return how Ketch will commit to the repository. - */ - public CommitMethod getCommitMethod() { - return commitMethod; - } - - /** - * Get when Ketch will commit to the repository. - * - * @return when Ketch will commit to the repository. - */ - public CommitSpeed getCommitSpeed() { - return commitSpeed; - } - - /** - * Called by leader to perform graceful shutdown. - *

    - * Default implementation cancels any scheduled retry. Subclasses may add - * additional logic before or after calling {@code super.shutdown()}. - *

    - * Called with {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held - * by caller. - */ - protected void shutdown() { - Future f = retryFuture; - if (f != null) { - retryFuture = null; - f.cancel(true); - } - } - - ReplicaSnapshot snapshot() { - ReplicaSnapshot s = new ReplicaSnapshot(this); - s.accepted = txnAccepted; - s.committed = txnCommitted; - s.state = state; - s.error = error; - s.retryAtMillis = waitingForRetry() ? retryAtMillis : 0; - return s; - } - - /** - * Update the leader's view of the replica after a poll. - *

    - * Called with {@link KetchLeader#lock} held by caller. - * - * @param refs - * map of refs from the replica. - */ - void initialize(Map refs) { - if (txnAccepted == null) { - txnAccepted = getId(refs.get(getSystem().getTxnAccepted())); - } - if (txnCommitted == null) { - txnCommitted = getId(refs.get(getSystem().getTxnCommitted())); - } - } - - ObjectId getTxnAccepted() { - return txnAccepted; - } - - boolean hasAccepted(LogIndex id) { - return equals(txnAccepted, id); - } - - private static boolean equals(@Nullable ObjectId a, LogIndex b) { - return a != null && b != null && AnyObjectId.isEqual(a, b); - } - - /** - * Schedule a proposal round with the replica. - *

    - * Called with {@link KetchLeader#lock} held by caller. - * - * @param round - * current round being run by the leader. - */ - void pushTxnAcceptedAsync(Round round) { - List cmds = new ArrayList<>(); - if (commitSpeed == BATCHED) { - LogIndex committedIndex = leader.getCommitted(); - if (equals(txnAccepted, committedIndex) - && !equals(txnCommitted, committedIndex)) { - prepareTxnCommitted(cmds, committedIndex); - } - } - - // TODO(sop) Lagging replicas should build accept on the fly. - if (round.stageCommands != null) { - for (ReceiveCommand cmd : round.stageCommands) { - // TODO(sop): Do not send certain object graphs to replica. - cmds.add(copy(cmd)); - } - } - cmds.add(new ReceiveCommand( - round.acceptedOldIndex, round.acceptedNewIndex, - getSystem().getTxnAccepted())); - pushAsync(new ReplicaPushRequest(this, cmds)); - } - - private static ReceiveCommand copy(ReceiveCommand c) { - return new ReceiveCommand(c.getOldId(), c.getNewId(), c.getRefName()); - } - - boolean shouldPushUnbatchedCommit(LogIndex committed, boolean leaderIdle) { - return (leaderIdle || commitSpeed == FAST) && hasAccepted(committed); - } - - void pushCommitAsync(LogIndex committed) { - List cmds = new ArrayList<>(); - prepareTxnCommitted(cmds, committed); - pushAsync(new ReplicaPushRequest(this, cmds)); - } - - private void prepareTxnCommitted(List cmds, - ObjectId committed) { - removeStaged(cmds, committed); - cmds.add(new ReceiveCommand( - txnCommitted, committed, - getSystem().getTxnCommitted())); - } - - private void removeStaged(List cmds, ObjectId committed) { - List a = staged.remove(committed); - if (a != null) { - delete(cmds, a); - } - if (staged.isEmpty() || !(committed instanceof LogIndex)) { - return; - } - - LogIndex committedIndex = (LogIndex) committed; - Iterator>> itr = staged - .entrySet().iterator(); - while (itr.hasNext()) { - Map.Entry> e = itr.next(); - if (e.getKey() instanceof LogIndex) { - LogIndex stagedIndex = (LogIndex) e.getKey(); - if (stagedIndex.isBefore(committedIndex)) { - delete(cmds, e.getValue()); - itr.remove(); - } - } - } - } - - private static void delete(List cmds, - List createCmds) { - for (ReceiveCommand cmd : createCmds) { - ObjectId id = cmd.getNewId(); - String name = cmd.getRefName(); - cmds.add(new ReceiveCommand(id, ObjectId.zeroId(), name)); - } - } - - /** - * Determine the next push for this replica (if any) and start it. - *

    - * If the replica has successfully accepted the committed state of the - * leader, this method will push all references to the replica using the - * configured {@link CommitMethod}. - *

    - * If the replica is {@link State#LAGGING} this method will begin catch up - * by sending a more recent {@code refs/txn/accepted}. - *

    - * Must be invoked with {@link KetchLeader#lock} held by caller. - */ - private void runNextPushRequest() { - LogIndex committed = leader.getCommitted(); - if (!equals(txnCommitted, committed) - && shouldPushUnbatchedCommit(committed, leader.isIdle())) { - pushCommitAsync(committed); - } - - if (queued.isEmpty() || !running.isEmpty() || waitingForRetry()) { - return; - } - - // Collapse all queued requests into a single request. - Map cmdMap = new HashMap<>(); - for (ReplicaPushRequest req : queued) { - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - ReceiveCommand old = cmdMap.remove(name); - if (old != null) { - cmd = new ReceiveCommand( - old.getOldId(), cmd.getNewId(), - name); - } - cmdMap.put(name, cmd); - } - } - queued.clear(); - waiting.clear(); - - List next = new ArrayList<>(cmdMap.values()); - for (ReceiveCommand cmd : next) { - running.put(cmd.getRefName(), cmd); - } - startPush(new ReplicaPushRequest(this, next)); - } - - private void pushAsync(ReplicaPushRequest req) { - if (defer(req)) { - // TODO(sop) Collapse during long retry outage. - for (ReceiveCommand cmd : req.getCommands()) { - waiting.put(cmd.getRefName(), cmd); - } - queued.add(req); - } else { - for (ReceiveCommand cmd : req.getCommands()) { - running.put(cmd.getRefName(), cmd); - } - startPush(req); - } - } - - private boolean defer(ReplicaPushRequest req) { - if (waitingForRetry()) { - // Prior communication failure; everything is deferred. - return true; - } - - for (ReceiveCommand nextCmd : req.getCommands()) { - ReceiveCommand priorCmd = waiting.get(nextCmd.getRefName()); - if (priorCmd == null) { - priorCmd = running.get(nextCmd.getRefName()); - } - if (priorCmd != null) { - // Another request pending on same ref; that must go first. - // Verify priorCmd.newId == nextCmd.oldId? - return true; - } - } - return false; - } - - private boolean waitingForRetry() { - Future f = retryFuture; - return f != null && !f.isDone(); - } - - private void retryLater(ReplicaPushRequest req) { - Collection cmds = req.getCommands(); - for (ReceiveCommand cmd : cmds) { - cmd.setResult(NOT_ATTEMPTED, null); - if (!waiting.containsKey(cmd.getRefName())) { - waiting.put(cmd.getRefName(), cmd); - } - } - queued.add(0, new ReplicaPushRequest(this, cmds)); - - if (!waitingForRetry()) { - long delay = FileUtils - .delay(lastRetryMillis, minRetryMillis, maxRetryMillis); - if (log.isDebugEnabled()) { - log.debug("Retrying {} after {} ms", //$NON-NLS-1$ - describeForLog(), Long.valueOf(delay)); - } - lastRetryMillis = delay; - retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay; - retryFuture = getSystem().getExecutor() - .schedule(new WeakRetryPush(this), delay, MILLISECONDS); - } - } - - /** Weakly holds a retrying replica, allowing it to garbage collect. */ - static class WeakRetryPush extends WeakReference - implements Callable { - WeakRetryPush(KetchReplica r) { - super(r); - } - - @Override - public Void call() throws Exception { - KetchReplica r = get(); - if (r != null) { - r.doRetryPush(); - } - return null; - } - } - - private void doRetryPush() { - leader.lock.lock(); - try { - retryFuture = null; - runNextPushRequest(); - } finally { - leader.lock.unlock(); - } - } - - /** - * Begin executing a single push. - *

    - * This method must move processing onto another thread. Called with - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held by caller. - * - * @param req - * the request to send to the replica. - */ - protected abstract void startPush(ReplicaPushRequest req); - - /** - * Callback from {@link ReplicaPushRequest} upon success or failure. - *

    - * Acquires the {@link KetchLeader#lock} and updates the leader's internal - * knowledge about this replica to reflect what has been learned during a - * push to the replica. In some cases of divergence this method may take - * some time to determine how the replica has diverged; to reduce contention - * this is evaluated before acquiring the leader lock. - * - * @param repo - * local repository instance used by the push thread. - * @param req - * push request just attempted. - */ - void afterPush(@Nullable Repository repo, ReplicaPushRequest req) { - ReceiveCommand acceptCmd = null; - ReceiveCommand commitCmd = null; - List stages = null; - - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - if (name.equals(getSystem().getTxnAccepted())) { - acceptCmd = cmd; - } else if (name.equals(getSystem().getTxnCommitted())) { - commitCmd = cmd; - } else if (cmd.getResult() == OK && cmd.getType() == CREATE - && name.startsWith(getSystem().getTxnStage())) { - if (stages == null) { - stages = new ArrayList<>(); - } - stages.add(cmd); - } - } - - State newState = null; - ObjectId acceptId = readId(req, acceptCmd); - if (repo != null && acceptCmd != null && acceptCmd.getResult() != OK - && req.getException() == null) { - try (LagCheck lag = new LagCheck(this, repo)) { - newState = lag.check(acceptId, acceptCmd); - acceptId = lag.getRemoteId(); - } - } - - leader.lock.lock(); - try { - for (ReceiveCommand cmd : req.getCommands()) { - running.remove(cmd.getRefName()); - } - - Throwable err = req.getException(); - if (err != null) { - state = OFFLINE; - error = err.toString(); - retryLater(req); - leader.onReplicaUpdate(this); - return; - } - - lastRetryMillis = 0; - error = null; - updateView(req, acceptId, commitCmd); - - if (acceptCmd != null && acceptCmd.getResult() == OK) { - state = hasAccepted(leader.getHead()) ? CURRENT : LAGGING; - if (stages != null) { - staged.put(acceptCmd.getNewId(), stages); - } - } else if (newState != null) { - state = newState; - } - - leader.onReplicaUpdate(this); - runNextPushRequest(); - } finally { - leader.lock.unlock(); - } - } - - private void updateView(ReplicaPushRequest req, @Nullable ObjectId acceptId, - ReceiveCommand commitCmd) { - if (acceptId != null) { - txnAccepted = acceptId; - } - - ObjectId committed = readId(req, commitCmd); - if (committed != null) { - txnCommitted = committed; - } else if (acceptId != null && txnCommitted == null) { - // Initialize during first conversation. - Map adv = req.getRefs(); - if (adv != null) { - Ref refs = adv.get(getSystem().getTxnCommitted()); - txnCommitted = getId(refs); - } - } - } - - @Nullable - private static ObjectId readId(ReplicaPushRequest req, - @Nullable ReceiveCommand cmd) { - if (cmd == null) { - // Ref was not in the command list, do not trust advertisement. - return null; - - } else if (cmd.getResult() == OK) { - // Currently at newId. - return cmd.getNewId(); - } - - Map refs = req.getRefs(); - return refs != null ? getId(refs.get(cmd.getRefName())) : null; - } - - /** - * Fetch objects from the remote using the calling thread. - *

    - * Called without {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock}. - * - * @param repo - * local repository to fetch objects into. - * @param req - * the request to fetch from a replica. - * @throws java.io.IOException - * communication with the replica was not possible. - */ - protected abstract void blockingFetch(Repository repo, - ReplicaFetchRequest req) throws IOException; - - /** - * Build a list of commands to commit - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS}. - * - * @param git - * local leader repository to read committed state from. - * @param current - * all known references in the replica's repository. Typically - * this comes from a push advertisement. - * @param committed - * state being pushed to {@code refs/txn/committed}. - * @return commands to update during commit. - * @throws java.io.IOException - * cannot read the committed state. - */ - protected Collection prepareCommit(Repository git, - Map current, ObjectId committed) throws IOException { - List delta = new ArrayList<>(); - Map remote = new HashMap<>(current); - try (RevWalk rw = new RevWalk(git); - TreeWalk tw = new TreeWalk(rw.getObjectReader())) { - tw.setRecursive(true); - tw.addTree(rw.parseCommit(committed).getTree()); - while (tw.next()) { - if (tw.getRawMode(0) != TYPE_GITLINK - || tw.isPathSuffix(PEEL, 2)) { - // Symbolic references cannot be pushed. - // Caching peeled values is handled remotely. - continue; - } - - // TODO(sop) Do not send certain ref names to replica. - String name = RefTree.refName(tw.getPathString()); - Ref oldRef = remote.remove(name); - ObjectId oldId = getId(oldRef); - ObjectId newId = tw.getObjectId(0); - if (!AnyObjectId.isEqual(oldId, newId)) { - delta.add(new ReceiveCommand(oldId, newId, name)); - } - } - } - - // Delete any extra references not in the committed state. - for (Ref ref : remote.values()) { - if (canDelete(ref)) { - delta.add(new ReceiveCommand( - ref.getObjectId(), ObjectId.zeroId(), - ref.getName())); - } - } - return delta; - } - - boolean canDelete(Ref ref) { - String name = ref.getName(); - if (HEAD.equals(name)) { - return false; - } - if (name.startsWith(getSystem().getTxnNamespace())) { - return false; - } - // TODO(sop) Do not delete precious names from replica. - return true; - } - - @NonNull - static ObjectId getId(@Nullable Ref ref) { - if (ref != null) { - ObjectId id = ref.getObjectId(); - if (id != null) { - return id; - } - } - return ObjectId.zeroId(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java deleted file mode 100644 index 8ad1d6033..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchConstants.ACCEPTED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.COMMITTED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_SECTION_KETCH; -import static org.eclipse.jgit.internal.ketch.KetchConstants.DEFAULT_TXN_NAMESPACE; -import static org.eclipse.jgit.internal.ketch.KetchConstants.STAGE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_NAME; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; - -import java.net.URISyntaxException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.URIish; -import org.eclipse.jgit.util.time.MonotonicClock; -import org.eclipse.jgit.util.time.MonotonicSystemClock; -import org.eclipse.jgit.util.time.ProposedTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Ketch system-wide configuration. - *

    - * This class provides useful defaults for testing and small proof of concepts. - * Full scale installations are expected to subclass and override methods to - * provide consistent configuration across all managed repositories. - *

    - * Servers should configure their own - * {@link java.util.concurrent.ScheduledExecutorService}. - */ -public class KetchSystem { - private static final Random RNG = new Random(); - - /** - * Get default executor, one thread per available processor. - * - * @return default executor, one thread per available processor. - */ - public static ScheduledExecutorService defaultExecutor() { - return DefaultExecutorHolder.I; - } - - private final ScheduledExecutorService executor; - private final MonotonicClock clock; - private final String txnNamespace; - private final String txnAccepted; - private final String txnCommitted; - private final String txnStage; - - /** - * Create a default system with a thread pool of 1 thread per CPU. - */ - public KetchSystem() { - this(defaultExecutor(), new MonotonicSystemClock(), DEFAULT_TXN_NAMESPACE); - } - - /** - * Create a Ketch system with the provided executor service. - * - * @param executor - * thread pool to run background operations. - * @param clock - * clock to create timestamps. - * @param txnNamespace - * reference namespace for the RefTree graph and associated - * transaction state. Must begin with {@code "refs/"} and end - * with {@code '/'}, for example {@code "refs/txn/"}. - */ - public KetchSystem(ScheduledExecutorService executor, MonotonicClock clock, - String txnNamespace) { - this.executor = executor; - this.clock = clock; - this.txnNamespace = txnNamespace; - this.txnAccepted = txnNamespace + ACCEPTED; - this.txnCommitted = txnNamespace + COMMITTED; - this.txnStage = txnNamespace + STAGE; - } - - /** - * Get executor to perform background operations. - * - * @return executor to perform background operations. - */ - public ScheduledExecutorService getExecutor() { - return executor; - } - - /** - * Get clock to obtain timestamps from. - * - * @return clock to obtain timestamps from. - */ - public MonotonicClock getClock() { - return clock; - } - - /** - * Get how long the leader will wait for the {@link #getClock()}'s - * {@code ProposedTimestamp} used in commits proposed to the RefTree graph - * ({@link #getTxnAccepted()}) - * - * @return how long the leader will wait for the {@link #getClock()}'s - * {@code ProposedTimestamp} used in commits proposed to the RefTree - * graph ({@link #getTxnAccepted()}). Defaults to 5 seconds. - */ - public Duration getMaxWaitForMonotonicClock() { - return Duration.ofSeconds(5); - } - - /** - * Whether elections should require monotonically increasing commit - * timestamps - * - * @return {@code true} if elections should require monotonically increasing - * commit timestamps. This requires a very good - * {@link org.eclipse.jgit.util.time.MonotonicClock}. - */ - public boolean requireMonotonicLeaderElections() { - return false; - } - - /** - * Get the namespace used for the RefTree graph and transaction management. - * - * @return reference namespace such as {@code "refs/txn/"}. - */ - public String getTxnNamespace() { - return txnNamespace; - } - - /** - * Get name of the accepted RefTree graph. - * - * @return name of the accepted RefTree graph. - */ - public String getTxnAccepted() { - return txnAccepted; - } - - /** - * Get name of the committed RefTree graph. - * - * @return name of the committed RefTree graph. - */ - public String getTxnCommitted() { - return txnCommitted; - } - - /** - * Get prefix for staged objects, e.g. {@code "refs/txn/stage/"}. - * - * @return prefix for staged objects, e.g. {@code "refs/txn/stage/"}. - */ - public String getTxnStage() { - return txnStage; - } - - /** - * Create new committer {@code PersonIdent} for ketch system - * - * @param time - * timestamp for the committer. - * @return identity line for the committer header of a RefTreeGraph. - */ - public PersonIdent newCommitter(ProposedTimestamp time) { - String name = "ketch"; //$NON-NLS-1$ - String email = "ketch@system"; //$NON-NLS-1$ - return new PersonIdent(name, email, time); - } - - /** - * Construct a random tag to identify a candidate during leader election. - *

    - * Multiple processes trying to elect themselves leaders at exactly the same - * time (rounded to seconds) using the same - * {@link #newCommitter(ProposedTimestamp)} identity strings, for the same - * term, may generate the same ObjectId for the election commit and falsely - * assume they have both won. - *

    - * Candidates add this tag to their election ballot commit to disambiguate - * the election. The tag only needs to be unique for a given triplet of - * {@link #newCommitter(ProposedTimestamp)}, system time (rounded to - * seconds), and term. If every replica in the system uses a unique - * {@code newCommitter} (such as including the host name after the - * {@code "@"} in the email address) the tag could be the empty string. - *

    - * The default implementation generates a few bytes of random data. - * - * @return unique tag; null or empty string if {@code newCommitter()} is - * sufficiently unique to identify the leader. - */ - @Nullable - public String newLeaderTag() { - int n = RNG.nextInt(1 << (6 * 4)); - return String.format("%06x", Integer.valueOf(n)); //$NON-NLS-1$ - } - - /** - * Construct the KetchLeader instance of a repository. - * - * @param repo - * local repository stored by the leader. - * @return leader instance. - * @throws java.net.URISyntaxException - * a follower configuration contains an unsupported URI. - */ - public KetchLeader createLeader(Repository repo) - throws URISyntaxException { - KetchLeader leader = new KetchLeader(this) { - @Override - protected Repository openRepository() { - repo.incrementOpen(); - return repo; - } - }; - leader.setReplicas(createReplicas(leader, repo)); - return leader; - } - - /** - * Get the collection of replicas for a repository. - *

    - * The collection of replicas must include the local repository. - * - * @param leader - * the leader driving these replicas. - * @param repo - * repository to get the replicas of. - * @return collection of replicas for the specified repository. - * @throws java.net.URISyntaxException - * a configured URI is invalid. - */ - protected List createReplicas(KetchLeader leader, - Repository repo) throws URISyntaxException { - List replicas = new ArrayList<>(); - Config cfg = repo.getConfig(); - String localName = getLocalName(cfg); - for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) { - if (!hasParticipation(cfg, name)) { - continue; - } - - ReplicaConfig kc = ReplicaConfig.newFromConfig(cfg, name); - if (name.equals(localName)) { - replicas.add(new LocalReplica(leader, name, kc)); - continue; - } - - RemoteConfig rc = new RemoteConfig(cfg, name); - List uris = rc.getPushURIs(); - if (uris.isEmpty()) { - uris = rc.getURIs(); - } - for (URIish uri : uris) { - String n = uris.size() == 1 ? name : uri.getHost(); - replicas.add(new RemoteGitReplica(leader, n, uri, kc, rc)); - } - } - return replicas; - } - - private static boolean hasParticipation(Config cfg, String name) { - return cfg.getString(CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE) != null; - } - - private static String getLocalName(Config cfg) { - return cfg.getString(CONFIG_SECTION_KETCH, null, CONFIG_KEY_NAME); - } - - static class DefaultExecutorHolder { - private static final Logger log = LoggerFactory.getLogger(KetchSystem.class); - static final ScheduledExecutorService I = create(); - - private static ScheduledExecutorService create() { - int cores = Runtime.getRuntime().availableProcessors(); - int threads = Math.max(5, cores); - log.info("Using {} threads", Integer.valueOf(threads)); //$NON-NLS-1$ - return Executors.newScheduledThreadPool( - threads, - new ThreadFactory() { - private final AtomicInteger threadCnt = new AtomicInteger(); - - @Override - public Thread newThread(Runnable r) { - int id = threadCnt.incrementAndGet(); - Thread thr = new Thread(r); - thr.setName("KetchExecutor-" + id); //$NON-NLS-1$ - return thr; - } - }); - } - - private DefaultExecutorHolder() { - } - } - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java deleted file mode 100644 index 6f9038bbd..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.nls.NLS; -import org.eclipse.jgit.nls.TranslationBundle; - -/** - * Translation bundle for the Ketch implementation. - */ -public class KetchText extends TranslationBundle { - /** - * Get an instance of this translation bundle. - * - * @return instance of this translation bundle. - */ - public static KetchText get() { - return NLS.getBundleFor(KetchText.class); - } - - // @formatter:off - /***/ public String accepted; - /***/ public String cannotFetchFromLocalReplica; - /***/ public String failed; - /***/ public String invalidFollowerUri; - /***/ public String leaderFailedToStore; - /***/ public String localReplicaRequired; - /***/ public String mismatchedTxnNamespace; - /***/ public String outsideTxnNamespace; - /***/ public String proposingUpdates; - /***/ public String queuedProposalFailedToApply; - /***/ public String starting; - /***/ public String unsupportedVoterCount; - /***/ public String waitingForQueue; -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java deleted file mode 100644 index 1f8384ff7..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.AHEAD; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.DIVERGENT; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; -import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * A helper to check if a {@link KetchReplica} is ahead or behind the leader. - */ -class LagCheck implements AutoCloseable { - private final KetchReplica replica; - private final Repository repo; - private RevWalk rw; - private ObjectId remoteId; - - LagCheck(KetchReplica replica, Repository repo) { - this.replica = replica; - this.repo = repo; - initRevWalk(); - } - - private void initRevWalk() { - if (rw != null) { - rw.close(); - } - - rw = new RevWalk(repo); - rw.setRetainBody(false); - } - - /** {@inheritDoc} */ - @Override - public void close() { - if (rw != null) { - rw.close(); - rw = null; - } - } - - ObjectId getRemoteId() { - return remoteId; - } - - KetchReplica.State check(ObjectId acceptId, ReceiveCommand acceptCmd) { - remoteId = acceptId; - if (remoteId == null) { - // Nothing advertised by the replica, value is unknown. - return UNKNOWN; - } - - if (AnyObjectId.isEqual(remoteId, ObjectId.zeroId())) { - // Replica does not have the txnAccepted reference. - return LAGGING; - } - - try { - RevCommit remote; - try { - remote = parseRemoteCommit(acceptCmd.getRefName()); - } catch (RefGoneException gone) { - // Replica does not have the txnAccepted reference. - return LAGGING; - } catch (MissingObjectException notFound) { - // Local repository does not know this commit so it cannot - // be including the replica's log. - return DIVERGENT; - } - - RevCommit head = rw.parseCommit(acceptCmd.getNewId()); - if (rw.isMergedInto(remote, head)) { - return LAGGING; - } - - // TODO(sop) Check term to see if my leader was deposed. - if (rw.isMergedInto(head, remote)) { - return AHEAD; - } - return DIVERGENT; - } catch (IOException err) { - KetchReplica.log.error(String.format( - "Cannot compare %s", //$NON-NLS-1$ - acceptCmd.getRefName()), err); - return UNKNOWN; - } - } - - private RevCommit parseRemoteCommit(String refName) - throws IOException, MissingObjectException, RefGoneException { - try { - return rw.parseCommit(remoteId); - } catch (MissingObjectException notLocal) { - // Fall through and try to acquire the object by fetching it. - } - - ReplicaFetchRequest fetch = new ReplicaFetchRequest( - Collections.singleton(refName), - Collections. emptySet()); - try { - replica.blockingFetch(repo, fetch); - } catch (IOException fetchErr) { - KetchReplica.log.error(String.format( - "Cannot fetch %s (%s) from %s", //$NON-NLS-1$ - remoteId.abbreviate(8).name(), refName, - replica.describeForLog()), fetchErr); - throw new MissingObjectException(remoteId, OBJ_COMMIT); - } - - Map adv = fetch.getRefs(); - if (adv == null) { - throw new MissingObjectException(remoteId, OBJ_COMMIT); - } - - Ref ref = adv.get(refName); - if (ref == null || ref.getObjectId() == null) { - throw new RefGoneException(); - } - - initRevWalk(); - remoteId = ref.getObjectId(); - return rw.parseCommit(remoteId); - } - - private static class RefGoneException extends Exception { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java deleted file mode 100644 index ce0672c16..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; - -/** - * A snapshot of a leader and its view of the world. - */ -public class LeaderSnapshot { - final List replicas = new ArrayList<>(); - KetchLeader.State state; - long term; - LogIndex headIndex; - LogIndex committedIndex; - boolean idle; - - LeaderSnapshot() { - } - - /** - * Get unmodifiable view of configured replicas. - * - * @return unmodifiable view of configured replicas. - */ - public Collection getReplicas() { - return Collections.unmodifiableList(replicas); - } - - /** - * Get current state of the leader. - * - * @return current state of the leader. - */ - public KetchLeader.State getState() { - return state; - } - - /** - * Whether the leader is not running a round to reach consensus, and has no - * rounds queued. - * - * @return {@code true} if the leader is not running a round to reach - * consensus, and has no rounds queued. - */ - public boolean isIdle() { - return idle; - } - - /** - * Get term of this leader - * - * @return term of this leader. Valid only if {@link #getState()} is - * currently - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER}. - */ - public long getTerm() { - return term; - } - - /** - * Get end of the leader's log - * - * @return end of the leader's log; null if leader hasn't started up enough - * to begin its own election. - */ - @Nullable - public LogIndex getHead() { - return headIndex; - } - - /** - * Get state the leader knows is committed on a majority of participant - * replicas - * - * @return state the leader knows is committed on a majority of participant - * replicas. Null until the leader instance has committed a log - * index within its own term. - */ - @Nullable - public LogIndex getCommitted() { - return committedIndex; - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - s.append(isIdle() ? "IDLE" : "RUNNING"); //$NON-NLS-1$ //$NON-NLS-2$ - s.append(" state ").append(getState()); //$NON-NLS-1$ - if (getTerm() > 0) { - s.append(" term ").append(getTerm()); //$NON-NLS-1$ - } - s.append('\n'); - s.append(String.format( - "%-10s %12s %12s\n", //$NON-NLS-1$ - "Replica", "Accepted", "Committed")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - s.append("------------------------------------\n"); //$NON-NLS-1$ - debug(s, "(leader)", getHead(), getCommitted()); //$NON-NLS-1$ - s.append('\n'); - for (ReplicaSnapshot r : getReplicas()) { - debug(s, r); - s.append('\n'); - } - s.append('\n'); - return s.toString(); - } - - private static void debug(StringBuilder b, ReplicaSnapshot s) { - KetchReplica replica = s.getReplica(); - debug(b, replica.getName(), s.getAccepted(), s.getCommitted()); - b.append(String.format(" %-8s %s", //$NON-NLS-1$ - replica.getParticipation(), s.getState())); - if (s.getState() == OFFLINE) { - String err = s.getErrorMessage(); - if (err != null) { - b.append(" (").append(err).append(')'); //$NON-NLS-1$ - } - } - } - - private static void debug(StringBuilder s, String name, - ObjectId accepted, ObjectId committed) { - s.append(String.format( - "%-10s %-12s %-12s", //$NON-NLS-1$ - name, str(accepted), str(committed))); - } - - static String str(ObjectId c) { - if (c instanceof LogIndex) { - return ((LogIndex) c).describeForLog(); - } else if (c != null) { - return c.abbreviate(8).name(); - } - return "-"; //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java deleted file mode 100644 index b2d59d77d..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.TXN_COMMITTED; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.MonotonicClock; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** - * Ketch replica running on the same system as the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. - */ -public class LocalReplica extends KetchReplica { - /** - * Configure a local replica. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this replica for debugging. - * @param cfg - * how Ketch should treat the local system. - */ - public LocalReplica(KetchLeader leader, String name, ReplicaConfig cfg) { - super(leader, name, cfg); - } - - /** {@inheritDoc} */ - @Override - protected String describeForLog() { - return String.format("%s (leader)", getName()); //$NON-NLS-1$ - } - - /** - * Initializes local replica by reading accepted and committed references. - *

    - * Loads accepted and committed references from the reference database of - * the local replica and stores their current ObjectIds in memory. - * - * @param repo - * repository to initialize state from. - * @throws IOException - * cannot read repository state. - */ - void initialize(Repository repo) throws IOException { - RefDatabase refdb = repo.getRefDatabase(); - if (refdb instanceof RefTreeDatabase) { - RefTreeDatabase treeDb = (RefTreeDatabase) refdb; - String txnNamespace = getSystem().getTxnNamespace(); - if (!txnNamespace.equals(treeDb.getTxnNamespace())) { - throw new IOException(MessageFormat.format( - KetchText.get().mismatchedTxnNamespace, - txnNamespace, treeDb.getTxnNamespace())); - } - refdb = treeDb.getBootstrap(); - } - initialize(refdb.exactRef( - getSystem().getTxnAccepted(), - getSystem().getTxnCommitted())); - } - - /** {@inheritDoc} */ - @Override - protected void startPush(ReplicaPushRequest req) { - getSystem().getExecutor().execute(() -> { - MonotonicClock clk = getSystem().getClock(); - try (Repository git = getLeader().openRepository(); - ProposedTimestamp ts = clk.propose()) { - try { - update(git, req, ts); - req.done(git); - } catch (Throwable err) { - req.setException(git, err); - } - } catch (IOException err) { - req.setException(null, err); - } - }); - } - - /** {@inheritDoc} */ - @Override - protected void blockingFetch(Repository repo, ReplicaFetchRequest req) - throws IOException { - throw new IOException(KetchText.get().cannotFetchFromLocalReplica); - } - - private void update(Repository git, ReplicaPushRequest req, - ProposedTimestamp ts) throws IOException { - RefDatabase refdb = git.getRefDatabase(); - CommitMethod method = getCommitMethod(); - - // Local replica probably uses RefTreeDatabase, the request should - // be only for the txnNamespace, so drop to the bootstrap layer. - if (refdb instanceof RefTreeDatabase) { - if (!isOnlyTxnNamespace(req.getCommands())) { - return; - } - - refdb = ((RefTreeDatabase) refdb).getBootstrap(); - method = TXN_COMMITTED; - } - - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addProposedTimestamp(ts); - batch.setRefLogIdent(getSystem().newCommitter(ts)); - batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ - batch.setAllowNonFastForwards(true); - - // RefDirectory updates multiple references sequentially. - // Run everything else first, then accepted (if present), - // then committed (if present). This ensures an earlier - // failure will not update these critical references. - ReceiveCommand accepted = null; - ReceiveCommand committed = null; - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - if (name.equals(getSystem().getTxnAccepted())) { - accepted = cmd; - } else if (name.equals(getSystem().getTxnCommitted())) { - committed = cmd; - } else { - batch.addCommand(cmd); - } - } - if (committed != null && method == ALL_REFS) { - Map refs = refdb.getRefs(ALL); - batch.addCommand(prepareCommit(git, refs, committed.getNewId())); - } - if (accepted != null) { - batch.addCommand(accepted); - } - if (committed != null) { - batch.addCommand(committed); - } - - try (RevWalk rw = new RevWalk(git)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - - // KetchReplica only cares about accepted and committed in - // advertisement. If they failed, store the current values - // back in the ReplicaPushRequest. - List failed = new ArrayList<>(2); - checkFailed(failed, accepted); - checkFailed(failed, committed); - if (!failed.isEmpty()) { - String[] arr = failed.toArray(new String[0]); - req.setRefs(refdb.exactRef(arr)); - } - } - - private static void checkFailed(List failed, ReceiveCommand cmd) { - if (cmd != null && cmd.getResult() != OK) { - failed.add(cmd.getRefName()); - } - } - - private boolean isOnlyTxnNamespace(Collection cmdList) { - // Be paranoid and reject non txnNamespace names, this - // is a programming error in Ketch that should not occur. - - String txnNamespace = getSystem().getTxnNamespace(); - for (ReceiveCommand cmd : cmdList) { - if (!cmd.getRefName().startsWith(txnNamespace)) { - cmd.setResult(REJECTED_OTHER_REASON, - MessageFormat.format( - KetchText.get().outsideTxnNamespace, - cmd.getRefName(), txnNamespace)); - ReceiveCommand.abort(cmdList); - return false; - } - } - return true; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java deleted file mode 100644 index ed65c06fa..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; - -/** - * An ObjectId for a commit extended with incrementing log index. - *

    - * For any two LogIndex instances, {@code A} is an ancestor of {@code C} - * reachable through parent edges in the graph if {@code A.index < C.index}. - * LogIndex provides a performance optimization for Ketch, the same information - * can be obtained from {@link org.eclipse.jgit.revwalk.RevWalk}. - *

    - * Index values are only valid within a single - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} instance after it has won - * an election. By restricting scope to a single leader new leaders do not need - * to traverse the entire history to determine the next {@code index} for new - * proposals. This differs from Raft, where leader election uses the log index - * and the term number to determine which replica holds a sufficiently - * up-to-date log. Since Ketch uses Git objects for storage of its replicated - * log, it keeps the term number as Raft does but uses standard Git operations - * to imply the log index. - *

    - * {@link org.eclipse.jgit.internal.ketch.Round#runAsync(AnyObjectId)} bumps the - * index as each new round is constructed. - */ -public class LogIndex extends ObjectId { - static LogIndex unknown(AnyObjectId id) { - return new LogIndex(id, 0); - } - - private final long index; - - private LogIndex(AnyObjectId id, long index) { - super(id); - this.index = index; - } - - LogIndex nextIndex(AnyObjectId id) { - return new LogIndex(id, index + 1); - } - - /** - * Get index provided by the current leader instance. - * - * @return index provided by the current leader instance. - */ - public long getIndex() { - return index; - } - - /** - * Check if this log position committed before another log position. - *

    - * Only valid for log positions in memory for the current leader. - * - * @param c - * other (more recent) log position. - * @return true if this log position was before {@code c} or equal to c and - * therefore any agreement of {@code c} implies agreement on this - * log position. - */ - boolean isBefore(LogIndex c) { - return index <= c.index; - } - - /** - * Create string suitable for debug logging containing the log index and - * abbreviated ObjectId. - * - * @return string suitable for debug logging containing the log index and - * abbreviated ObjectId. - */ - @SuppressWarnings("boxing") - public String describeForLog() { - return String.format("%5d/%s", index, abbreviate(6).name()); //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @SuppressWarnings("boxing") - @Override - public String toString() { - return String.format("LogId[%5d/%s]", index, name()); //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java deleted file mode 100644 index ca27281a8..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.Proposal.State.ABORTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.NEW; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.storage.reftree.Command; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.PushCertificate; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** - * A proposal to be applied in a Ketch system. - *

    - * Pushing to a Ketch leader results in the leader making a proposal. The - * proposal includes the list of reference updates. The leader attempts to send - * the proposal to a quorum of replicas by pushing the proposal to a "staging" - * area under the {@code refs/txn/stage/} namespace. If the proposal succeeds - * then the changes are durable and the leader can commit the proposal. - *

    - * Proposals are executed by - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#queueProposal(Proposal)}, - * which runs them asynchronously in the background. Proposals are thread-safe - * futures allowing callers to {@link #await()} for results or be notified by - * callback using {@link #addListener(Runnable)}. - */ -public class Proposal { - /** Current state of the proposal. */ - public enum State { - /** Proposal has not yet been given to a {@link KetchLeader}. */ - NEW(false), - - /** - * Proposal was validated and has entered the queue, but a round - * containing this proposal has not started yet. - */ - QUEUED(false), - - /** Round containing the proposal has begun and is in progress. */ - RUNNING(false), - - /** - * Proposal was executed through a round. Individual results from - * {@link Proposal#getCommands()}, {@link Command#getResult()} explain - * the success or failure outcome. - */ - EXECUTED(true), - - /** Proposal was aborted and did not reach consensus. */ - ABORTED(true); - - private final boolean done; - - private State(boolean done) { - this.done = done; - } - - /** @return true if this is a terminal state. */ - public boolean isDone() { - return done; - } - } - - private final List commands; - private PersonIdent author; - private String message; - private PushCertificate pushCert; - - private List timestamps; - private final List listeners = new CopyOnWriteArrayList<>(); - private final AtomicReference state = new AtomicReference<>(NEW); - - /** - * Create a proposal from a list of Ketch commands. - * - * @param cmds - * prepared list of commands. - */ - public Proposal(List cmds) { - commands = Collections.unmodifiableList(new ArrayList<>(cmds)); - } - - /** - * Create a proposal from a collection of received commands. - * - * @param rw - * walker to assist in preparing commands. - * @param cmds - * list of pending commands. - * @throws org.eclipse.jgit.errors.MissingObjectException - * newId of a command is not found locally. - * @throws java.io.IOException - * local objects cannot be accessed. - */ - public Proposal(RevWalk rw, Collection cmds) - throws MissingObjectException, IOException { - commands = asCommandList(rw, cmds); - } - - private static List asCommandList(RevWalk rw, - Collection cmds) - throws MissingObjectException, IOException { - List commands = new ArrayList<>(cmds.size()); - for (ReceiveCommand cmd : cmds) { - commands.add(new Command(rw, cmd)); - } - return Collections.unmodifiableList(commands); - } - - /** - * Get commands from this proposal. - * - * @return commands from this proposal. - */ - public Collection getCommands() { - return commands; - } - - /** - * Get optional author of the proposal. - * - * @return optional author of the proposal. - */ - @Nullable - public PersonIdent getAuthor() { - return author; - } - - /** - * Set the author for the proposal. - * - * @param who - * optional identity of the author of the proposal. - * @return {@code this} - */ - public Proposal setAuthor(@Nullable PersonIdent who) { - author = who; - return this; - } - - /** - * Get optional message for the commit log of the RefTree. - * - * @return optional message for the commit log of the RefTree. - */ - @Nullable - public String getMessage() { - return message; - } - - /** - * Set the message to appear in the commit log of the RefTree. - * - * @param msg - * message text for the commit. - * @return {@code this} - */ - public Proposal setMessage(@Nullable String msg) { - message = msg != null && !msg.isEmpty() ? msg : null; - return this; - } - - /** - * Get optional certificate signing the references. - * - * @return optional certificate signing the references. - */ - @Nullable - public PushCertificate getPushCertificate() { - return pushCert; - } - - /** - * Set the push certificate signing the references. - * - * @param cert - * certificate, may be null. - * @return {@code this} - */ - public Proposal setPushCertificate(@Nullable PushCertificate cert) { - pushCert = cert; - return this; - } - - /** - * Get timestamps that Ketch must block for. - * - * @return timestamps that Ketch must block for. These may have been used as - * commit times inside the objects involved in the proposal. - */ - public List getProposedTimestamps() { - if (timestamps != null) { - return timestamps; - } - return Collections.emptyList(); - } - - /** - * Request the proposal to wait for the affected timestamps to resolve. - * - * @param ts - * a {@link org.eclipse.jgit.util.time.ProposedTimestamp} object. - * @return {@code this}. - */ - public Proposal addProposedTimestamp(ProposedTimestamp ts) { - if (timestamps == null) { - timestamps = new ArrayList<>(4); - } - timestamps.add(ts); - return this; - } - - /** - * Add a callback to be invoked when the proposal is done. - *

    - * A proposal is done when it has entered either - * {@link org.eclipse.jgit.internal.ketch.Proposal.State#EXECUTED} or - * {@link org.eclipse.jgit.internal.ketch.Proposal.State#ABORTED} state. If - * the proposal is already done {@code callback.run()} is immediately - * invoked on the caller's thread. - * - * @param callback - * method to run after the proposal is done. The callback may be - * run on a Ketch system thread and should be completed quickly. - */ - public void addListener(Runnable callback) { - boolean runNow = false; - synchronized (state) { - if (state.get().isDone()) { - runNow = true; - } else { - listeners.add(callback); - } - } - if (runNow) { - callback.run(); - } - } - - /** Set command result as OK. */ - void success() { - for (Command c : commands) { - if (c.getResult() == NOT_ATTEMPTED) { - c.setResult(OK); - } - } - notifyState(EXECUTED); - } - - /** Mark commands as "transaction aborted". */ - void abort() { - Command.abort(commands, null); - notifyState(ABORTED); - } - - /** - * Read the current state of the proposal. - * - * @return read the current state of the proposal. - */ - public State getState() { - return state.get(); - } - - /** - * Whether the proposal was attempted - * - * @return {@code true} if the proposal was attempted. A true value does not - * mean consensus was reached, only that the proposal was considered - * and will not be making any more progress beyond its current - * state. - */ - public boolean isDone() { - return state.get().isDone(); - } - - /** - * Wait for the proposal to be attempted and {@link #isDone()} to be true. - * - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public void await() throws InterruptedException { - synchronized (state) { - while (!state.get().isDone()) { - state.wait(); - } - } - } - - /** - * Wait for the proposal to be attempted and {@link #isDone()} to be true. - * - * @param wait - * how long to wait. - * @param unit - * unit describing the wait time. - * @return true if the proposal is done; false if the method timed out. - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public boolean await(long wait, TimeUnit unit) throws InterruptedException { - synchronized (state) { - if (state.get().isDone()) { - return true; - } - state.wait(unit.toMillis(wait)); - return state.get().isDone(); - } - } - - /** - * Wait for the proposal to exit a state. - * - * @param notIn - * state the proposal should not be in to return. - * @param wait - * how long to wait. - * @param unit - * unit describing the wait time. - * @return true if the proposal exited the state; false on time out. - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public boolean awaitStateChange(State notIn, long wait, TimeUnit unit) - throws InterruptedException { - synchronized (state) { - if (state.get() != notIn) { - return true; - } - state.wait(unit.toMillis(wait)); - return state.get() != notIn; - } - } - - void notifyState(State s) { - synchronized (state) { - state.set(s); - state.notifyAll(); - } - if (s.isDone()) { - for (Runnable callback : listeners) { - callback.run(); - } - listeners.clear(); - } - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - s.append("Ketch Proposal {\n"); //$NON-NLS-1$ - s.append(" ").append(state.get()).append('\n'); //$NON-NLS-1$ - if (author != null) { - s.append(" author ").append(author).append('\n'); //$NON-NLS-1$ - } - if (message != null) { - s.append(" message ").append(message).append('\n'); //$NON-NLS-1$ - } - for (Command c : commands) { - s.append(" "); //$NON-NLS-1$ - format(s, c.getOldRef(), "CREATE"); //$NON-NLS-1$ - s.append(' '); - format(s, c.getNewRef(), "DELETE"); //$NON-NLS-1$ - s.append(' ').append(c.getRefName()); - if (c.getResult() != ReceiveCommand.Result.NOT_ATTEMPTED) { - s.append(' ').append(c.getResult()); // $NON-NLS-1$ - } - s.append('\n'); - } - s.append('}'); - return s.toString(); - } - - private static void format(StringBuilder s, @Nullable Ref r, String n) { - if (r == null) { - s.append(n); - } else if (r.isSymbolic()) { - s.append(r.getTarget().getName()); - } else { - ObjectId id = r.getObjectId(); - if (id != null) { - s.append(id.abbreviate(8).name()); - } - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java deleted file mode 100644 index b73183abd..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.Proposal.State.RUNNING; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.storage.reftree.Command; -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** A {@link Round} that aggregates and sends user {@link Proposal}s. */ -class ProposalRound extends Round { - private final List todo; - private RefTree queuedTree; - - ProposalRound(KetchLeader leader, LogIndex head, List todo, - @Nullable RefTree tree) { - super(leader, head); - this.todo = todo; - - if (tree != null && canCombine(todo)) { - this.queuedTree = tree; - } else { - leader.roundHoldsReferenceToRefTree = false; - } - } - - private static boolean canCombine(List todo) { - Proposal first = todo.get(0); - for (int i = 1; i < todo.size(); i++) { - if (!canCombine(first, todo.get(i))) { - return false; - } - } - return true; - } - - private static boolean canCombine(Proposal a, Proposal b) { - String aMsg = nullToEmpty(a.getMessage()); - String bMsg = nullToEmpty(b.getMessage()); - return aMsg.equals(bMsg) && canCombine(a.getAuthor(), b.getAuthor()); - } - - private static String nullToEmpty(@Nullable String str) { - return str != null ? str : ""; //$NON-NLS-1$ - } - - private static boolean canCombine(@Nullable PersonIdent a, - @Nullable PersonIdent b) { - if (a != null && b != null) { - // Same name and email address. Combine timestamp as the two - // proposals are running concurrently and appear together or - // not at all from the point of view of an outside reader. - return a.getName().equals(b.getName()) - && a.getEmailAddress().equals(b.getEmailAddress()); - } - - // If a and b are null, both will be the system identity. - return a == null && b == null; - } - - @Override - void start() throws IOException { - for (Proposal p : todo) { - p.notifyState(RUNNING); - } - try { - ObjectId id; - try (Repository git = leader.openRepository(); - ProposedTimestamp ts = getSystem().getClock().propose()) { - id = insertProposals(git, ts); - blockUntil(ts); - } - runAsync(id); - } catch (NoOp e) { - for (Proposal p : todo) { - p.success(); - } - leader.lock.lock(); - try { - leader.nextRound(); - } finally { - leader.lock.unlock(); - } - } catch (IOException e) { - abort(); - throw e; - } - } - - private ObjectId insertProposals(Repository git, ProposedTimestamp ts) - throws IOException, NoOp { - ObjectId id; - try (ObjectInserter inserter = git.newObjectInserter()) { - // TODO(sop) Process signed push certificates. - - if (queuedTree != null) { - id = insertSingleProposal(git, ts, inserter); - } else { - id = insertMultiProposal(git, ts, inserter); - } - - stageCommands = makeStageList(git, inserter); - inserter.flush(); - } - return id; - } - - private ObjectId insertSingleProposal(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException, NoOp { - // Fast path: tree is passed in with all proposals applied. - ObjectId treeId = queuedTree.writeTree(inserter); - queuedTree = null; - leader.roundHoldsReferenceToRefTree = false; - - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(acceptedOldIndex); - if (treeId.equals(c.getTree())) { - throw new NoOp(); - } - } - } - - Proposal p = todo.get(0); - CommitBuilder b = new CommitBuilder(); - b.setTreeId(treeId); - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - b.setParentId(acceptedOldIndex); - } - b.setCommitter(leader.getSystem().newCommitter(ts)); - b.setAuthor(p.getAuthor() != null ? p.getAuthor() : b.getCommitter()); - b.setMessage(message(p)); - return inserter.insert(b); - } - - private ObjectId insertMultiProposal(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException, NoOp { - // The tree was not passed in, or there are multiple proposals - // each needing their own commit. Reset the tree and replay each - // proposal in order as individual commits. - ObjectId lastIndex = acceptedOldIndex; - ObjectId oldTreeId; - RefTree tree; - if (ObjectId.zeroId().equals(lastIndex)) { - oldTreeId = ObjectId.zeroId(); - tree = RefTree.newEmptyTree(); - } else { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(lastIndex); - oldTreeId = c.getTree(); - tree = RefTree.read(rw.getObjectReader(), c.getTree()); - } - } - - PersonIdent committer = leader.getSystem().newCommitter(ts); - for (Proposal p : todo) { - if (!tree.apply(p.getCommands())) { - // This should not occur, previously during queuing the - // commands were successfully applied to the pending tree. - // Abort the entire round. - throw new IOException( - KetchText.get().queuedProposalFailedToApply); - } - - ObjectId treeId = tree.writeTree(inserter); - if (treeId.equals(oldTreeId)) { - continue; - } - - CommitBuilder b = new CommitBuilder(); - b.setTreeId(treeId); - if (!ObjectId.zeroId().equals(lastIndex)) { - b.setParentId(lastIndex); - } - b.setAuthor(p.getAuthor() != null ? p.getAuthor() : committer); - b.setCommitter(committer); - b.setMessage(message(p)); - lastIndex = inserter.insert(b); - } - if (lastIndex.equals(acceptedOldIndex)) { - throw new NoOp(); - } - return lastIndex; - } - - private String message(Proposal p) { - StringBuilder m = new StringBuilder(); - String msg = p.getMessage(); - if (msg != null && !msg.isEmpty()) { - m.append(msg); - while (m.length() < 2 || m.charAt(m.length() - 2) != '\n' - || m.charAt(m.length() - 1) != '\n') { - m.append('\n'); - } - } - m.append(KetchConstants.TERM.getName()) - .append(": ") //$NON-NLS-1$ - .append(leader.getTerm()); - return m.toString(); - } - - void abort() { - for (Proposal p : todo) { - p.abort(); - } - } - - @Override - void success() { - for (Proposal p : todo) { - p.success(); - } - } - - private List makeStageList(Repository git, - ObjectInserter inserter) throws IOException { - // For each branch, collapse consecutive updates to only most recent, - // avoiding sending multiple objects in a rapid fast-forward chain, or - // rewritten content. - Map byRef = new HashMap<>(); - for (Proposal p : todo) { - for (Command c : p.getCommands()) { - Ref n = c.getNewRef(); - if (n != null && !n.isSymbolic()) { - byRef.put(n.getName(), n.getObjectId()); - } - } - } - if (byRef.isEmpty()) { - return Collections.emptyList(); - } - - Set newObjs = new HashSet<>(byRef.values()); - StageBuilder b = new StageBuilder( - leader.getSystem().getTxnStage(), - acceptedNewIndex); - return b.makeStageList(newObjs, git, inserter); - } - - private void blockUntil(ProposedTimestamp ts) - throws TimeIsUncertainException { - List times = todo.stream() - .flatMap(p -> p.getProposedTimestamps().stream()) - .collect(Collectors.toCollection(ArrayList::new)); - times.add(ts); - - try { - Duration maxWait = getSystem().getMaxWaitForMonotonicClock(); - ProposedTimestamp.blockUntil(times, maxWait); - } catch (InterruptedException | TimeoutException e) { - throw new TimeIsUncertainException(e); - } - } - - private static class NoOp extends Exception { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java deleted file mode 100644 index fac93c84b..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NODELETE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.NotSupportedException; -import org.eclipse.jgit.errors.TransportException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.FetchConnection; -import org.eclipse.jgit.transport.PushConnection; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.RemoteRefUpdate; -import org.eclipse.jgit.transport.Transport; -import org.eclipse.jgit.transport.URIish; - -/** - * Representation of a Git repository on a remote replica system. - *

    - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} will contact the replica - * using the Git wire protocol. - *

    - * The remote replica may be fully Ketch-aware, or a standard Git server. - */ -public class RemoteGitReplica extends KetchReplica { - private final URIish uri; - private final RemoteConfig remoteConfig; - - /** - * Configure a new remote. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this remote for debugging. - * @param uri - * URI to connect to the follower's repository. - * @param cfg - * how Ketch should treat the remote system. - * @param rc - * optional remote configuration describing how to contact the - * peer repository. - */ - public RemoteGitReplica(KetchLeader leader, String name, URIish uri, - ReplicaConfig cfg, @Nullable RemoteConfig rc) { - super(leader, name, cfg); - this.uri = uri; - this.remoteConfig = rc; - } - - /** - * Get URI to contact the remote peer repository. - * - * @return URI to contact the remote peer repository. - */ - public URIish getURI() { - return uri; - } - - /** - * Get optional configuration describing how to contact the peer. - * - * @return optional configuration describing how to contact the peer. - */ - @Nullable - protected RemoteConfig getRemoteConfig() { - return remoteConfig; - } - - /** {@inheritDoc} */ - @Override - protected String describeForLog() { - return String.format("%s @ %s", getName(), getURI()); //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @Override - protected void startPush(ReplicaPushRequest req) { - getSystem().getExecutor().execute(() -> { - try (Repository git = getLeader().openRepository()) { - try { - push(git, req); - req.done(git); - } catch (Throwable err) { - req.setException(git, err); - } - } catch (IOException err) { - req.setException(null, err); - } - }); - } - - private void push(Repository repo, ReplicaPushRequest req) - throws NotSupportedException, TransportException, IOException { - Map adv; - List cmds = asUpdateList(req.getCommands()); - try (Transport transport = Transport.open(repo, uri)) { - RemoteConfig rc = getRemoteConfig(); - if (rc != null) { - transport.applyConfig(rc); - } - transport.setPushAtomic(true); - adv = push(repo, transport, cmds); - } - for (RemoteCommand c : cmds) { - c.copyStatusToResult(); - } - req.setRefs(adv); - } - - private Map push(Repository git, Transport transport, - List cmds) throws IOException { - Map updates = asUpdateMap(cmds); - try (PushConnection connection = transport.openPush()) { - Map adv = connection.getRefsMap(); - RemoteRefUpdate accepted = updates.get(getSystem().getTxnAccepted()); - if (accepted != null && !isExpectedValue(adv, accepted)) { - abort(cmds); - return adv; - } - - RemoteRefUpdate committed = updates.get(getSystem().getTxnCommitted()); - if (committed != null && !isExpectedValue(adv, committed)) { - abort(cmds); - return adv; - } - if (committed != null && getCommitMethod() == ALL_REFS) { - prepareCommit(git, cmds, updates, adv, - committed.getNewObjectId()); - } - - connection.push(NullProgressMonitor.INSTANCE, updates); - return adv; - } - } - - private static boolean isExpectedValue(Map adv, - RemoteRefUpdate u) { - Ref r = adv.get(u.getRemoteName()); - if (!AnyObjectId.isEqual(getId(r), u.getExpectedOldObjectId())) { - ((RemoteCommand) u).cmd.setResult(LOCK_FAILURE); - return false; - } - return true; - } - - private void prepareCommit(Repository git, List cmds, - Map updates, Map adv, - ObjectId committed) throws IOException { - for (ReceiveCommand cmd : prepareCommit(git, adv, committed)) { - RemoteCommand c = new RemoteCommand(cmd); - cmds.add(c); - updates.put(c.getRemoteName(), c); - } - } - - private static List asUpdateList( - Collection cmds) { - try { - List toPush = new ArrayList<>(cmds.size()); - for (ReceiveCommand cmd : cmds) { - toPush.add(new RemoteCommand(cmd)); - } - return toPush; - } catch (IOException e) { - // Cannot occur as no IO was required to build the command. - throw new IllegalStateException(e); - } - } - - private static Map asUpdateMap( - List cmds) { - Map m = new LinkedHashMap<>(); - for (RemoteCommand cmd : cmds) { - m.put(cmd.getRemoteName(), cmd); - } - return m; - } - - private static void abort(List cmds) { - List tmp = new ArrayList<>(cmds.size()); - for (RemoteCommand cmd : cmds) { - tmp.add(cmd.cmd); - } - ReceiveCommand.abort(tmp); - } - - /** {@inheritDoc} */ - @Override - protected void blockingFetch(Repository repo, ReplicaFetchRequest req) - throws NotSupportedException, TransportException { - try (Transport transport = Transport.open(repo, uri)) { - RemoteConfig rc = getRemoteConfig(); - if (rc != null) { - transport.applyConfig(rc); - } - fetch(transport, req); - } - } - - private void fetch(Transport transport, ReplicaFetchRequest req) - throws NotSupportedException, TransportException { - try (FetchConnection conn = transport.openFetch()) { - Map remoteRefs = conn.getRefsMap(); - req.setRefs(remoteRefs); - - List want = new ArrayList<>(); - for (String name : req.getWantRefs()) { - Ref ref = remoteRefs.get(name); - if (ref != null && ref.getObjectId() != null) { - want.add(ref); - } - } - for (ObjectId id : req.getWantObjects()) { - want.add(new ObjectIdRef.Unpeeled(NETWORK, id.name(), id)); - } - - conn.fetch(NullProgressMonitor.INSTANCE, want, - Collections. emptySet()); - } - } - - static class RemoteCommand extends RemoteRefUpdate { - final ReceiveCommand cmd; - - RemoteCommand(ReceiveCommand cmd) throws IOException { - super(null, null, - cmd.getNewId(), cmd.getRefName(), - true /* force update */, - null /* no local tracking ref */, - cmd.getOldId()); - this.cmd = cmd; - } - - void copyStatusToResult() { - if (cmd.getResult() == NOT_ATTEMPTED) { - switch (getStatus()) { - case OK: - case UP_TO_DATE: - case NON_EXISTING: - cmd.setResult(OK); - break; - - case REJECTED_NODELETE: - cmd.setResult(REJECTED_NODELETE); - break; - - case REJECTED_NONFASTFORWARD: - cmd.setResult(REJECTED_NONFASTFORWARD); - break; - - case REJECTED_OTHER_REASON: - cmd.setResult(REJECTED_OTHER_REASON, getMessage()); - break; - - default: - cmd.setResult(REJECTED_OTHER_REASON, getStatus().name()); - break; - } - } - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java deleted file mode 100644 index 1d323b849..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_COMMIT; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_SPEED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod; -import org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed; -import org.eclipse.jgit.internal.ketch.KetchReplica.Participation; -import org.eclipse.jgit.lib.Config; - -/** - * Configures a {@link org.eclipse.jgit.internal.ketch.KetchReplica}. - */ -public class ReplicaConfig { - /** - * Read a configuration from a config block. - * - * @param cfg - * configuration to read. - * @param name - * of the replica being configured. - * @return replica configuration for {@code name}. - */ - public static ReplicaConfig newFromConfig(Config cfg, String name) { - return new ReplicaConfig().fromConfig(cfg, name); - } - - private Participation participation = Participation.FULL; - private CommitMethod commitMethod = CommitMethod.ALL_REFS; - private CommitSpeed commitSpeed = CommitSpeed.BATCHED; - private long minRetry = SECONDS.toMillis(5); - private long maxRetry = MINUTES.toMillis(1); - - /** - * Get participation of the replica in the system. - * - * @return participation of the replica in the system. - */ - public Participation getParticipation() { - return participation; - } - - /** - * Get how Ketch should apply committed changes. - * - * @return how Ketch should apply committed changes. - */ - public CommitMethod getCommitMethod() { - return commitMethod; - } - - /** - * Get how quickly should Ketch commit. - * - * @return how quickly should Ketch commit. - */ - public CommitSpeed getCommitSpeed() { - return commitSpeed; - } - - /** - * Returns the minimum wait delay before retrying a failure. - * - * @param unit - * to get retry delay in. - * @return minimum delay before retrying a failure. - */ - public long getMinRetry(TimeUnit unit) { - return unit.convert(minRetry, MILLISECONDS); - } - - /** - * Returns the maximum wait delay before retrying a failure. - * - * @param unit - * to get retry delay in. - * @return maximum delay before retrying a failure. - */ - public long getMaxRetry(TimeUnit unit) { - return unit.convert(maxRetry, MILLISECONDS); - } - - /** - * Update the configuration from a config block. - * - * @param cfg - * configuration to read. - * @param name - * of the replica being configured. - * @return {@code this} - */ - public ReplicaConfig fromConfig(Config cfg, String name) { - participation = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE, - participation); - commitMethod = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_COMMIT, - commitMethod); - commitSpeed = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_SPEED, - commitSpeed); - minRetry = getMillis(cfg, name, "ketch-minRetry", minRetry); //$NON-NLS-1$ - maxRetry = getMillis(cfg, name, "ketch-maxRetry", maxRetry); //$NON-NLS-1$ - return this; - } - - private static long getMillis(Config cfg, String name, String key, - long defaultValue) { - String valStr = cfg.getString(CONFIG_KEY_REMOTE, name, key); - if (valStr == null) { - return defaultValue; - } - - valStr = valStr.trim(); - if (valStr.isEmpty()) { - return defaultValue; - } - - Matcher m = UnitMap.PATTERN.matcher(valStr); - if (!m.matches()) { - return defaultValue; - } - - String digits = m.group(1); - String unitName = m.group(2).trim(); - TimeUnit unit = UnitMap.UNITS.get(unitName); - if (unit == null) { - return defaultValue; - } - - try { - if (digits.indexOf('.') == -1) { - return unit.toMillis(Long.parseLong(digits)); - } - - double val = Double.parseDouble(digits); - return (long) (val * unit.toMillis(1)); - } catch (NumberFormatException nfe) { - return defaultValue; - } - } - - static class UnitMap { - static final Pattern PATTERN = Pattern - .compile("^([1-9][0-9]*(?:\\.[0-9]*)?)\\s*(.*)$"); //$NON-NLS-1$ - - static final Map UNITS; - - static { - Map m = new HashMap<>(); - TimeUnit u = MILLISECONDS; - m.put("", u); //$NON-NLS-1$ - m.put("ms", u); //$NON-NLS-1$ - m.put("millis", u); //$NON-NLS-1$ - m.put("millisecond", u); //$NON-NLS-1$ - m.put("milliseconds", u); //$NON-NLS-1$ - - u = SECONDS; - m.put("s", u); //$NON-NLS-1$ - m.put("sec", u); //$NON-NLS-1$ - m.put("secs", u); //$NON-NLS-1$ - m.put("second", u); //$NON-NLS-1$ - m.put("seconds", u); //$NON-NLS-1$ - - u = MINUTES; - m.put("m", u); //$NON-NLS-1$ - m.put("min", u); //$NON-NLS-1$ - m.put("mins", u); //$NON-NLS-1$ - m.put("minute", u); //$NON-NLS-1$ - m.put("minutes", u); //$NON-NLS-1$ - - u = HOURS; - m.put("h", u); //$NON-NLS-1$ - m.put("hr", u); //$NON-NLS-1$ - m.put("hrs", u); //$NON-NLS-1$ - m.put("hour", u); //$NON-NLS-1$ - m.put("hours", u); //$NON-NLS-1$ - - u = DAYS; - m.put("d", u); //$NON-NLS-1$ - m.put("day", u); //$NON-NLS-1$ - m.put("days", u); //$NON-NLS-1$ - - UNITS = Collections.unmodifiableMap(m); - } - - private UnitMap() { - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java deleted file mode 100644 index f50ad62c8..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Map; -import java.util.Set; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; - -/** - * A fetch request to obtain objects from a replica, and its result. - */ -public class ReplicaFetchRequest { - private final Set wantRefs; - private final Set wantObjects; - private Map refs; - - /** - * Construct a new fetch request for a replica. - * - * @param wantRefs - * named references to be fetched. - * @param wantObjects - * specific objects to be fetched. - */ - public ReplicaFetchRequest(Set wantRefs, - Set wantObjects) { - this.wantRefs = wantRefs; - this.wantObjects = wantObjects; - } - - /** - * Get references to be fetched. - * - * @return references to be fetched. - */ - public Set getWantRefs() { - return wantRefs; - } - - /** - * Get objects to be fetched. - * - * @return objects to be fetched. - */ - public Set getWantObjects() { - return wantObjects; - } - - /** - * Get remote references, usually from the advertisement. - * - * @return remote references, usually from the advertisement. - */ - @Nullable - public Map getRefs() { - return refs; - } - - /** - * Set references observed from the replica. - * - * @param refs - * references observed from the replica. - */ - public void setRefs(Map refs) { - this.refs = refs; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java deleted file mode 100644 index 273760bc6..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Collection; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * A push request sending objects to a replica, and its result. - *

    - * Implementors of {@link org.eclipse.jgit.internal.ketch.KetchReplica} must - * populate the command result fields, {@link #setRefs(Map)}, and call one of - * {@link #setException(Repository, Throwable)} or {@link #done(Repository)} to - * finish processing. - */ -public class ReplicaPushRequest { - private final KetchReplica replica; - private final Collection commands; - private Map refs; - private Throwable exception; - private boolean notified; - - /** - * Construct a new push request for a replica. - * - * @param replica - * the replica being pushed to. - * @param commands - * commands to be executed. - */ - public ReplicaPushRequest(KetchReplica replica, - Collection commands) { - this.replica = replica; - this.commands = commands; - } - - /** - * Get commands to be executed, and their results. - * - * @return commands to be executed, and their results. - */ - public Collection getCommands() { - return commands; - } - - /** - * Get remote references, usually from the advertisement. - * - * @return remote references, usually from the advertisement. - */ - @Nullable - public Map getRefs() { - return refs; - } - - /** - * Set references observed from the replica. - * - * @param refs - * references observed from the replica. - */ - public void setRefs(Map refs) { - this.refs = refs; - } - - /** - * Get exception thrown, if any. - * - * @return exception thrown, if any. - */ - @Nullable - public Throwable getException() { - return exception; - } - - /** - * Mark the request as crashing with a communication error. - *

    - * This method may take significant time acquiring the leader lock and - * updating the Ketch state machine with the failure. - * - * @param repo - * local repository reference used by the push attempt. - * @param err - * exception thrown during communication. - */ - public void setException(@Nullable Repository repo, Throwable err) { - if (KetchReplica.log.isErrorEnabled()) { - KetchReplica.log.error(describe("failed"), err); //$NON-NLS-1$ - } - if (!notified) { - notified = true; - exception = err; - replica.afterPush(repo, this); - } - } - - /** - * Mark the request as completed without exception. - *

    - * This method may take significant time acquiring the leader lock and - * updating the Ketch state machine with results from this replica. - * - * @param repo - * local repository reference used by the push attempt. - */ - public void done(Repository repo) { - if (KetchReplica.log.isDebugEnabled()) { - KetchReplica.log.debug(describe("completed")); //$NON-NLS-1$ - } - if (!notified) { - notified = true; - replica.afterPush(repo, this); - } - } - - private String describe(String heading) { - StringBuilder b = new StringBuilder(); - b.append("push to "); //$NON-NLS-1$ - b.append(replica.describeForLog()); - b.append(' ').append(heading).append(":\n"); //$NON-NLS-1$ - for (ReceiveCommand cmd : commands) { - b.append(String.format( - " %-12s %-12s %s %s", //$NON-NLS-1$ - LeaderSnapshot.str(cmd.getOldId()), - LeaderSnapshot.str(cmd.getNewId()), - cmd.getRefName(), - cmd.getResult())); - if (cmd.getMessage() != null) { - b.append(' ').append(cmd.getMessage()); - } - b.append('\n'); - } - return b.toString(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java deleted file mode 100644 index 05e4ed693..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Date; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; - -/** - * A snapshot of a replica. - * - * @see LeaderSnapshot - */ -public class ReplicaSnapshot { - final KetchReplica replica; - ObjectId accepted; - ObjectId committed; - KetchReplica.State state; - String error; - long retryAtMillis; - - ReplicaSnapshot(KetchReplica replica) { - this.replica = replica; - } - - /** - * Get the replica this snapshot describes the state of - * - * @return the replica this snapshot describes the state of - */ - public KetchReplica getReplica() { - return replica; - } - - /** - * Get current state of the replica - * - * @return current state of the replica - */ - public KetchReplica.State getState() { - return state; - } - - /** - * Get last known Git commit at {@code refs/txn/accepted} - * - * @return last known Git commit at {@code refs/txn/accepted} - */ - @Nullable - public ObjectId getAccepted() { - return accepted; - } - - /** - * Get last known Git commit at {@code refs/txn/committed} - * - * @return last known Git commit at {@code refs/txn/committed} - */ - @Nullable - public ObjectId getCommitted() { - return committed; - } - - /** - * Get error message - * - * @return if {@link #getState()} == - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.State#OFFLINE} - * an optional human-readable message from the transport system - * explaining the failure. - */ - @Nullable - public String getErrorMessage() { - return error; - } - - /** - * Get when the leader will retry communication with the offline or lagging - * replica - * - * @return time (usually in the future) when the leader will retry - * communication with the offline or lagging replica; null if no - * retry is scheduled or necessary. - */ - @Nullable - public Date getRetryAt() { - return retryAtMillis > 0 ? new Date(retryAtMillis) : null; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java deleted file mode 100644 index 05da5be05..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.io.IOException; -import java.util.List; - -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * One round-trip to all replicas proposing a log entry. - *

    - * In Raft a log entry represents a state transition at a specific index in the - * replicated log. The leader can only append log entries to the log. - *

    - * In Ketch a log entry is recorded under the {@code refs/txn} namespace. This - * occurs when: - *

      - *
    • a replica wants to establish itself as a new leader by proposing a new - * term (see {@link ElectionRound}) - *
    • an established leader wants to gain consensus on new {@link Proposal}s - * (see {@link ProposalRound}) - *
    - */ -abstract class Round { - final KetchLeader leader; - final LogIndex acceptedOldIndex; - LogIndex acceptedNewIndex; - List stageCommands; - - Round(KetchLeader leader, LogIndex head) { - this.leader = leader; - this.acceptedOldIndex = head; - } - - KetchSystem getSystem() { - return leader.getSystem(); - } - - /** - * Creates a commit for {@code refs/txn/accepted} and calls - * {@link #runAsync(AnyObjectId)} to begin execution of the round across - * the system. - *

    - * If references are being updated (such as in a {@link ProposalRound}) the - * RefTree may be modified. - *

    - * Invoked without {@link KetchLeader#lock} to build objects. - * - * @throws IOException - * the round cannot build new objects within the leader's - * repository. The leader may be unable to execute. - */ - abstract void start() throws IOException; - - /** - * Asynchronously distribute the round's new value for - * {@code refs/txn/accepted} to all replicas. - *

    - * Invoked by {@link #start()} after new commits have been created for the - * log. The method passes {@code newId} to {@link KetchLeader} to be - * distributed to all known replicas. - * - * @param newId - * new value for {@code refs/txn/accepted}. - */ - void runAsync(AnyObjectId newId) { - acceptedNewIndex = acceptedOldIndex.nextIndex(newId); - leader.runAsync(this); - } - - /** - * Notify the round it was accepted by a majority of the system. - *

    - * Invoked by the leader with {@link KetchLeader#lock} held by the caller. - */ - abstract void success(); -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java deleted file mode 100644 index 40d86e1a8..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.treewalk.EmptyTreeIterator; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.treewalk.filter.TreeFilter; - -/** - * Constructs a set of commands to stage content during a proposal. - */ -public class StageBuilder { - /** - * Acceptable number of references to send in a single stage transaction. - *

    - * If the number of unique objects exceeds this amount the builder will - * attempt to decrease the reference count by chaining commits.. - */ - private static final int SMALL_BATCH_SIZE = 5; - - /** - * Acceptable number of commits to chain together using parent pointers. - *

    - * When staging many unique commits the {@link StageBuilder} batches - * together unrelated commits as parents of a temporary commit. After the - * proposal completes the temporary commit is discarded and can be garbage - * collected by all replicas. - */ - private static final int TEMP_PARENT_BATCH_SIZE = 128; - - private static final byte[] PEEL = { ' ', '^' }; - - private final String txnStage; - private final String txnId; - - /** - * Construct a stage builder for a transaction. - * - * @param txnStageNamespace - * namespace for transaction references to build - * {@code "txnStageNamespace/txnId.n"} style names. - * @param txnId - * identifier used to name temporary staging refs. - */ - public StageBuilder(String txnStageNamespace, ObjectId txnId) { - this.txnStage = txnStageNamespace; - this.txnId = txnId.name(); - } - - /** - * Compare two RefTrees and return commands to stage new objects. - *

    - * This method ignores the lineage between the two RefTrees and does a - * straight diff on the two trees. New objects will be staged. The diff - * strategy is useful to catch-up a lagging replica, without sending every - * intermediate step. This may mean the replica does not have the same - * object set as other replicas if there are rewinds or branch deletes. - * - * @param git - * source repository to read {@code oldTree} and {@code newTree} - * from. - * @param oldTree - * accepted RefTree on the replica ({@code refs/txn/accepted}). - * Use {@link org.eclipse.jgit.lib.ObjectId#zeroId()} if the - * remote does not have any ref tree, e.g. a new replica catching - * up. - * @param newTree - * RefTree being sent to the replica. The trees will be compared. - * @return list of commands to create {@code "refs/txn/stage/..."} - * references on replicas anchoring new objects into the repository - * while a transaction gains consensus. - * @throws java.io.IOException - * {@code git} cannot be accessed to compare {@code oldTree} and - * {@code newTree} to build the object set. - */ - public List makeStageList(Repository git, ObjectId oldTree, - ObjectId newTree) throws IOException { - try (RevWalk rw = new RevWalk(git); - TreeWalk tw = new TreeWalk(rw.getObjectReader()); - ObjectInserter ins = git.newObjectInserter()) { - if (AnyObjectId.isEqual(oldTree, ObjectId.zeroId())) { - tw.addTree(new EmptyTreeIterator()); - } else { - tw.addTree(rw.parseTree(oldTree)); - } - tw.addTree(rw.parseTree(newTree)); - tw.setFilter(TreeFilter.ANY_DIFF); - tw.setRecursive(true); - - Set newObjs = new HashSet<>(); - while (tw.next()) { - if (tw.getRawMode(1) == TYPE_GITLINK - && !tw.isPathSuffix(PEEL, 2)) { - newObjs.add(tw.getObjectId(1)); - } - } - - List cmds = makeStageList(newObjs, git, ins); - ins.flush(); - return cmds; - } - } - - /** - * Construct a set of commands to stage objects on a replica. - * - * @param newObjs - * objects to send to a replica. - * @param git - * local repository to read source objects from. Required to - * perform minification of {@code newObjs}. - * @param inserter - * inserter to write temporary commit objects during minification - * if many new branches are created by {@code newObjs}. - * @return list of commands to create {@code "refs/txn/stage/..."} - * references on replicas anchoring {@code newObjs} into the - * repository while a transaction gains consensus. - * @throws java.io.IOException - * {@code git} cannot be accessed to perform minification of - * {@code newObjs}. - */ - public List makeStageList(Set newObjs, - @Nullable Repository git, @Nullable ObjectInserter inserter) - throws IOException { - if (git == null || newObjs.size() <= SMALL_BATCH_SIZE) { - // Without a source repository can only construct unique set. - List cmds = new ArrayList<>(newObjs.size()); - for (ObjectId id : newObjs) { - stage(cmds, id); - } - return cmds; - } - - List cmds = new ArrayList<>(); - List commits = new ArrayList<>(); - reduceObjects(cmds, commits, git, newObjs); - - if (inserter == null || commits.size() <= 1 - || (cmds.size() + commits.size()) <= SMALL_BATCH_SIZE) { - // Without an inserter to aggregate commits, or for a small set of - // commits just send one stage ref per commit. - for (RevCommit c : commits) { - stage(cmds, c.copy()); - } - return cmds; - } - - // 'commits' is sorted most recent to least recent commit. - // Group batches of commits and build a chain. - // TODO(sop) Cluster by restricted graphs to support filtering. - ObjectId tip = null; - for (int end = commits.size(); end > 0;) { - int start = Math.max(0, end - TEMP_PARENT_BATCH_SIZE); - List batch = commits.subList(start, end); - List parents = new ArrayList<>(1 + batch.size()); - if (tip != null) { - parents.add(tip); - } - parents.addAll(batch); - - CommitBuilder b = new CommitBuilder(); - b.setTreeId(batch.get(0).getTree()); - b.setParentIds(parents); - b.setAuthor(tmpAuthor(batch)); - b.setCommitter(b.getAuthor()); - tip = inserter.insert(b); - end = start; - } - stage(cmds, tip); - return cmds; - } - - private static PersonIdent tmpAuthor(List commits) { - // Construct a predictable author using most recent commit time. - int t = 0; - for (int i = 0; i < commits.size();) { - t = Math.max(t, commits.get(i).getCommitTime()); - } - String name = "Ketch Stage"; //$NON-NLS-1$ - String email = "tmp@tmp"; //$NON-NLS-1$ - return new PersonIdent(name, email, t * 1000L, 0); - } - - private void reduceObjects(List cmds, - List commits, Repository git, - Set newObjs) throws IOException { - try (RevWalk rw = new RevWalk(git)) { - rw.setRetainBody(false); - - for (ObjectId id : newObjs) { - RevObject obj = rw.parseAny(id); - if (obj instanceof RevCommit) { - rw.markStart((RevCommit) obj); - } else { - stage(cmds, id); - } - } - - for (RevCommit c; (c = rw.next()) != null;) { - commits.add(c); - rw.markUninteresting(c); - } - } - } - - private void stage(List cmds, ObjectId id) { - int estLen = txnStage.length() + txnId.length() + 5; - StringBuilder n = new StringBuilder(estLen); - n.append(txnStage).append(txnId).append('.'); - n.append(Integer.toHexString(cmds.size())); - cmds.add(new ReceiveCommand(ObjectId.zeroId(), id, n.toString())); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java deleted file mode 100644 index f665e6a43..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.io.IOException; - -import org.eclipse.jgit.internal.JGitText; - -class TimeIsUncertainException extends IOException { - private static final long serialVersionUID = 1L; - - TimeIsUncertainException() { - super(JGitText.get().timeIsUncertain); - } - - TimeIsUncertainException(Exception e) { - super(JGitText.get().timeIsUncertain, e); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java deleted file mode 100644 index dfe03752c..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Distributed consensus system built on Git. - */ -package org.eclipse.jgit.internal.ketch; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityChecker.java similarity index 88% rename from org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityChecker.java index 89aef7dc4..d8056490a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.util.ArrayList; @@ -21,12 +21,16 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; +import org.eclipse.jgit.revwalk.BitmapWalker; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevObject; /** * Checks if all objects are reachable from certain starting points using * bitmaps. */ -class BitmappedObjectReachabilityChecker +public class BitmappedObjectReachabilityChecker implements ObjectReachabilityChecker { private final ObjectWalk walk; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityChecker.java similarity index 92% rename from org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityChecker.java index 0d9c4593b..37721ad1e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.util.ArrayList; @@ -23,12 +23,17 @@ import org.eclipse.jgit.lib.BitmapIndex.Bitmap; import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevFlag; +import org.eclipse.jgit.revwalk.RevSort; +import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.RevFilter; /** * Checks the reachability using bitmaps. */ -class BitmappedReachabilityChecker implements ReachabilityChecker { +public class BitmappedReachabilityChecker implements ReachabilityChecker { private final RevWalk walk; @@ -42,7 +47,7 @@ class BitmappedReachabilityChecker implements ReachabilityChecker { * @throws IOException * if the index or the object reader cannot be opened. */ - BitmappedReachabilityChecker(RevWalk walk) + public BitmappedReachabilityChecker(RevWalk walk) throws IOException { this.walk = walk; if (walk.getObjectReader().getBitmapIndex() == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityChecker.java similarity index 82% rename from org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityChecker.java index df5d68a66..1d1f5fdda 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.io.InvalidObjectException; @@ -17,12 +17,18 @@ import java.util.stream.Stream; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevSort; /** * Checks if all objects are reachable from certain starting points doing a * walk. */ -class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker { +public class PedestrianObjectReachabilityChecker + implements ObjectReachabilityChecker { private final ObjectWalk walk; /** @@ -31,7 +37,7 @@ class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker { * @param walk * ObjectWalk instance to reuse. Caller retains ownership. */ - PedestrianObjectReachabilityChecker(ObjectWalk walk) { + public PedestrianObjectReachabilityChecker(ObjectWalk walk) { this.walk = walk; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityChecker.java similarity index 84% rename from org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianReachabilityChecker.java rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityChecker.java index 5dc03776c..a03306b6e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.util.Collection; @@ -17,12 +17,16 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevSort; +import org.eclipse.jgit.revwalk.RevWalk; /** * Checks the reachability walking the graph from the starters towards the * target. */ -class PedestrianReachabilityChecker implements ReachabilityChecker { +public class PedestrianReachabilityChecker implements ReachabilityChecker { private final boolean topoSort; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 876cbec16..26d5b5b17 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -13,7 +13,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -45,7 +44,6 @@ import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; -import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -95,7 +93,6 @@ public class DfsGarbageCollector { private Set allHeadsAndTags; private Set allTags; private Set nonHeads; - private Set txnHeads; private Set tagTargets; /** @@ -318,7 +315,6 @@ public boolean pack(ProgressMonitor pm) throws IOException { allHeadsAndTags = new HashSet<>(); allTags = new HashSet<>(); nonHeads = new HashSet<>(); - txnHeads = new HashSet<>(); tagTargets = new HashSet<>(); for (Ref ref : refsBefore) { if (ref.isSymbolic() || ref.getObjectId() == null) { @@ -328,8 +324,6 @@ public boolean pack(ProgressMonitor pm) throws IOException { allHeads.add(ref.getObjectId()); } else if (isTag(ref)) { allTags.add(ref.getObjectId()); - } else if (RefTreeNames.isRefTree(refdb, ref.getName())) { - txnHeads.add(ref.getObjectId()); } else { nonHeads.add(ref.getObjectId()); } @@ -355,7 +349,6 @@ public boolean pack(ProgressMonitor pm) throws IOException { try { packHeads(pm); packRest(pm); - packRefTreeGraph(pm); packGarbage(pm); objdb.commitPack(newPackDesc, toPrune()); rollback = false; @@ -559,19 +552,6 @@ private void packRest(ProgressMonitor pm) throws IOException { } } - private void packRefTreeGraph(ProgressMonitor pm) throws IOException { - if (txnHeads.isEmpty()) - return; - - try (PackWriter pw = newPackWriter()) { - for (ObjectIdSet packedObjs : newPackObj) - pw.excludeObjects(packedObjs); - pw.preparePack(pm, txnHeads, NONE); - if (0 < pw.getObjectCount()) - writePack(GC_TXN, pw, pm, 0 /* unknown pack size */); - } - } - private void packGarbage(ProgressMonitor pm) throws IOException { PackConfig cfg = new PackConfig(packConfig); cfg.setReuseDeltas(true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 4dab3b20c..46ec87df5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -104,13 +104,6 @@ public enum PackSource { /** Created from non-heads by {@link DfsGarbageCollector}. */ GC_REST, - /** - * RefTreeGraph pack was created by Git garbage collection. - * - * @see DfsGarbageCollector - */ - GC_TXN, - /** * Pack was created by Git garbage collection. *

    @@ -133,7 +126,6 @@ public enum PackSource { .add(COMPACT) .add(GC) .add(GC_REST) - .add(GC_TXN) .add(UNREACHABLE_GARBAGE) .build(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java index 3f113a3ee..8e124e3c2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java @@ -48,7 +48,6 @@ public boolean wasDeltaAttempted() { switch (pack.getPackDescription().getPackSource()) { case GC: case GC_REST: - case GC_TXN: return true; default: return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java index 0c8755fca..4f418ab4d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java @@ -533,7 +533,6 @@ static boolean isGC(PackSource s) { switch (s) { case GC: case GC_REST: - case GC_TXN: return true; default: return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index b1e95520c..96ca690c1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -607,8 +607,15 @@ private IOException packfileIsTruncated() { private void readFully(long position, byte[] dstbuf, int dstoff, int cnt, DfsReader ctx) throws IOException { - if (ctx.copy(this, position, dstbuf, dstoff, cnt) != cnt) - throw new EOFException(); + while (cnt > 0) { + int copied = ctx.copy(this, position, dstbuf, dstoff, cnt); + if (copied == 0) { + throw new EOFException(); + } + position += copied; + dstoff += copied; + cnt -= copied; + } } ObjectLoader load(DfsReader ctx, long pos) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java index 8a54431d5..6c3b056ef 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -17,6 +17,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.eclipse.jgit.annotations.Nullable; @@ -62,11 +63,12 @@ protected DfsReftableDatabase(DfsRepository repo) { reftableDatabase = new ReftableDatabase() { @Override public MergedReftable openMergedReftable() throws IOException { - DfsReftableDatabase.this.getLock().lock(); + Lock l = DfsReftableDatabase.this.getLock(); + l.lock(); try { return new MergedReftable(stack().readers()); } finally { - DfsReftableDatabase.this.getLock().unlock(); + l.unlock(); } } }; @@ -174,6 +176,13 @@ public List getRefsByPrefix(String prefix) throws IOException { return reftableDatabase.getRefsByPrefix(prefix); } + /** {@inheritDoc} */ + @Override + public List getRefsByPrefixWithExclusions(String include, Set excludes) + throws IOException { + return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); + } + /** {@inheritDoc} */ @Override public Set getTipsWithSha1(ObjectId id) throws IOException { @@ -207,7 +216,8 @@ boolean exists() throws IOException { @Override void clearCache() { - getLock().lock(); + ReentrantLock l = getLock(); + l.lock(); try { if (ctx != null) { ctx.close(); @@ -219,7 +229,7 @@ void clearCache() { stack = null; } } finally { - getLock().unlock(); + l.unlock(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java index 45d9c85c8..103653542 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java @@ -25,7 +25,7 @@ final class ByteArrayWindow extends ByteWindow { private final byte[] array; - ByteArrayWindow(PackFile pack, long o, byte[] b) { + ByteArrayWindow(Pack pack, long o, byte[] b) { super(pack, o, b.length); array = b; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java index 870321632..b6877578c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java @@ -27,7 +27,7 @@ final class ByteBufferWindow extends ByteWindow { private final ByteBuffer buffer; - ByteBufferWindow(PackFile pack, long o, ByteBuffer b) { + ByteBufferWindow(Pack pack, long o, ByteBuffer b) { super(pack, o, b.capacity()); buffer = b; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java index 159f31c97..31e7eadd8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java @@ -27,7 +27,7 @@ *

    */ abstract class ByteWindow { - protected final PackFile pack; + protected final Pack pack; protected final long start; @@ -37,13 +37,13 @@ abstract class ByteWindow { * Constructor for ByteWindow. * * @param p - * a {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * a {@link org.eclipse.jgit.internal.storage.file.Pack}. * @param s * where the byte window starts in the pack file * @param n * size of the byte window */ - protected ByteWindow(PackFile p, long s, int n) { + protected ByteWindow(Pack p, long s, int n) { pack = p; start = s; end = start + n; @@ -53,8 +53,8 @@ final int size() { return (int) (end - start); } - final boolean contains(PackFile neededFile, long neededPos) { - return pack == neededFile && start <= neededPos && neededPos < end; + final boolean contains(Pack neededPack, long neededPos) { + return pack == neededPack && start <= neededPos && neededPos < end; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java index 9c7a2e711..7dedeb57a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java @@ -239,7 +239,7 @@ InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId objectId, } @Override - PackFile openPack(File pack) throws IOException { + Pack openPack(File pack) throws IOException { return wrapped.openPack(pack); } @@ -250,7 +250,7 @@ void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, } @Override - Collection getPacks() { + Collection getPacks() { return wrapped.getPacks(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java index cef5a330f..69cebadf1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java @@ -49,7 +49,7 @@ static void reconfigure(WindowCacheConfig cfg) { cache = new Slot[CACHE_SZ]; } - Entry get(PackFile pack, long position) { + Entry get(Pack pack, long position) { Slot e = cache[hash(position)]; if (e == null) return null; @@ -63,7 +63,7 @@ Entry get(PackFile pack, long position) { return null; } - void store(final PackFile pack, final long position, + void store(final Pack pack, final long position, final byte[] data, final int objectType) { if (data.length > maxByteCount) return; // Too large to cache. @@ -146,7 +146,7 @@ private static class Slot { Slot lruNext; - PackFile provider; + Pack provider; long position; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java index 11ed10c90..01dd27d9f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java @@ -71,7 +71,7 @@ abstract ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id) abstract InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException; - abstract PackFile openPack(File pack) throws IOException; + abstract Pack openPack(File pack) throws IOException; - abstract Collection getPacks(); + abstract Collection getPacks(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index e613a5806..a80fa837b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -21,7 +21,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -107,12 +109,13 @@ public boolean hasFastTipsWithSha1() throws IOException { * @throws IOException on I/O errors */ public void compactFully() throws IOException { - reftableDatabase.getLock().lock(); + Lock l = reftableDatabase.getLock(); + l.lock(); try { reftableStack.compactFully(); reftableDatabase.clearCache(); } finally { - reftableDatabase.getLock().unlock(); + l.unlock(); } } @@ -177,6 +180,13 @@ public Map getRefs(String prefix) throws IOException { RefList.emptyList()); } + /** {@inheritDoc} */ + @Override + public List getRefsByPrefixWithExclusions(String include, Set excludes) + throws IOException { + return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); + } + /** {@inheritDoc} */ @Override public List getAdditionalRefs() throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java index bc2039c56..b5e3927bc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java @@ -21,6 +21,7 @@ import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -65,6 +66,8 @@ private static class StackEntry { private final Runnable onChange; + private final SecureRandom random = new SecureRandom(); + private final Supplier configSupplier; // Used for stats & testing. @@ -365,8 +368,9 @@ private long nextUpdateIndex() throws IOException { } private String filename(long low, long high) { - return String.format("%012x-%012x", //$NON-NLS-1$ - Long.valueOf(low), Long.valueOf(high)); + return String.format("%012x-%012x-%08x", //$NON-NLS-1$ + Long.valueOf(low), Long.valueOf(high), + Integer.valueOf(random.nextInt())); } /** @@ -636,6 +640,9 @@ public int hashCode() { @Override public boolean equals(Object other) { + if (other == null) { + return false; + } Segment o = (Segment) other; return o.bytes == bytes && o.log == log && o.start == start && o.end == end; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index fd052cec2..fecced1ae 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -40,7 +40,6 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.lib.BaseRepositoryBuilder; import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.ConfigConstants; @@ -182,9 +181,6 @@ public FileRepository(BaseRepositoryBuilder options) throws IOException { if (StringUtils.equalsIgnoreCase(reftype, ConfigConstants.CONFIG_REF_STORAGE_REFTABLE)) { refs = new FileReftableDatabase(this); - } else if (StringUtils.equalsIgnoreCase(reftype, - ConfigConstants.CONFIG_REFSTORAGE_REFTREE)) { - refs = new RefTreeDatabase(this, new RefDirectory(this)); } else { throw new IOException(JGitText.get().unknownRepositoryFormat); } @@ -247,7 +243,7 @@ && getDirectory().getName().startsWith(".")) //$NON-NLS-1$ RefUpdate head = updateRef(Constants.HEAD); head.disableRefLog(); - head.link(Constants.R_HEADS + Constants.MASTER); + head.link(Constants.R_HEADS + getInitialBranch()); final boolean fileMode; if (getFS().supportsExecute()) { @@ -640,7 +636,7 @@ void convertToPackedRefs(boolean writeLogs, boolean backup) throws IOException { refsHeadsFile.delete(); // RefDirectory wants to create the refs/ directory from scratch, so // remove that too. - refsFile.delete(); + refsFile.delete(); // remove HEAD so its previous invalid value doesn't cause issues. headFile.delete(); @@ -668,7 +664,7 @@ void convertToPackedRefs(boolean writeLogs, boolean backup) throws IOException { for (ReflogEntry e : logs) { logWriter.log(r.getName(), e); } - } + } } try (RevWalk rw = new RevWalk(this)) { @@ -731,7 +727,7 @@ void convertToReftable(boolean writeLogs, boolean backup) throws IOException { File reftableDir = new File(getDirectory(), Constants.REFTABLE); File headFile = new File(getDirectory(), Constants.HEAD); - if (reftableDir.exists() && reftableDir.listFiles().length > 0) { + if (reftableDir.exists() && FileUtils.hasFiles(reftableDir.toPath())) { throw new IOException(JGitText.get().reftableDirExists); } @@ -763,9 +759,11 @@ void convertToReftable(boolean writeLogs, boolean backup) } } else { FileUtils.delete(packedRefs, FileUtils.SKIP_MISSING); - FileUtils.delete(headFile); - FileUtils.delete(logsDir, FileUtils.RECURSIVE); - FileUtils.delete(refsFile, FileUtils.RECURSIVE); + FileUtils.delete(headFile, FileUtils.SKIP_MISSING); + FileUtils.delete(logsDir, + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); + FileUtils.delete(refsFile, + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); for (String r : additional) { new File(getDirectory(), r).delete(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 54ff7d29c..8c48cd94b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -12,8 +12,10 @@ import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_FILESTORE_ATTRIBUTES; import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_TIMESTAMP_RESOLUTION; + import java.io.File; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; @@ -221,14 +223,20 @@ protected FileSnapshot(File file, boolean useConfig) { this.file = file; this.lastRead = Instant.now(); this.fileStoreAttributeCache = useConfig - ? FS.getFileStoreAttributes(file.toPath().getParent()) + ? FS.getFileStoreAttributes(file.toPath()) : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(file); + } catch (NoSuchFileException e) { + this.lastModified = Instant.EPOCH; + this.size = 0L; + this.fileKey = MISSING_FILEKEY; + return; } catch (IOException e) { - this.lastModified = Instant.ofEpochMilli(file.lastModified()); - this.size = file.length(); + LOG.error(e.getMessage(), e); + this.lastModified = Instant.EPOCH; + this.size = 0L; this.fileKey = MISSING_FILEKEY; return; } @@ -309,9 +317,14 @@ public boolean isModified(File path) { currLastModified = fileAttributes.lastModifiedTime().toInstant(); currSize = fileAttributes.size(); currFileKey = getFileKey(fileAttributes); + } catch (NoSuchFileException e) { + currLastModified = Instant.EPOCH; + currSize = 0L; + currFileKey = MISSING_FILEKEY; } catch (IOException e) { - currLastModified = Instant.ofEpochMilli(path.lastModified()); - currSize = path.length(); + LOG.error(e.getMessage(), e); + currLastModified = Instant.EPOCH; + currSize = 0L; currFileKey = MISSING_FILEKEY; } sizeChanged = isSizeChanged(currSize); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 1f2fe1057..75de3be89 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -60,7 +60,6 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; @@ -206,16 +205,16 @@ public GC(FileRepository repo) { * gc.log. * * @return the collection of - * {@link org.eclipse.jgit.internal.storage.file.PackFile}'s which + * {@link org.eclipse.jgit.internal.storage.file.Pack}'s which * are newly created * @throws java.io.IOException * @throws java.text.ParseException * If the configuration parameter "gc.pruneexpire" couldn't be * parsed */ - // TODO(ms): change signature and return Future> + // TODO(ms): change signature and return Future> @SuppressWarnings("FutureReturnValueIgnored") - public Collection gc() throws IOException, ParseException { + public Collection gc() throws IOException, ParseException { if (!background) { return doGc(); } @@ -225,9 +224,9 @@ public Collection gc() throws IOException, ParseException { return Collections.emptyList(); } - Callable> gcTask = () -> { + Callable> gcTask = () -> { try { - Collection newPacks = doGc(); + Collection newPacks = doGc(); if (automatic && tooManyLooseObjects()) { String message = JGitText.get().gcTooManyUnpruned; gcLog.write(message); @@ -259,14 +258,14 @@ private ExecutorService executor() { return (executor != null) ? executor : WorkQueue.getExecutor(); } - private Collection doGc() throws IOException, ParseException { + private Collection doGc() throws IOException, ParseException { if (automatic && !needGc()) { return Collections.emptyList(); } pm.start(6 /* tasks */); packRefs(); // TODO: implement reflog_expire(pm, repo); - Collection newPacks = repack(); + Collection newPacks = repack(); prune(Collections.emptySet()); // TODO: implement rerere_gc(pm); return newPacks; @@ -282,7 +281,7 @@ private Collection doGc() throws IOException, ParseException { * @param existing * @throws IOException */ - private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet existing) + private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, Pack pack, HashSet existing) throws IOException { for (PackIndex.MutableEntry entry : pack) { ObjectId oid = entry.toObjectId(); @@ -314,10 +313,10 @@ private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackF * @throws ParseException * @throws IOException */ - private void deleteOldPacks(Collection oldPacks, - Collection newPacks) throws ParseException, IOException { + private void deleteOldPacks(Collection oldPacks, + Collection newPacks) throws ParseException, IOException { HashSet ids = new HashSet<>(); - for (PackFile pack : newPacks) { + for (Pack pack : newPacks) { for (PackIndex.MutableEntry entry : pack) { ids.add(entry.toObjectId()); } @@ -330,12 +329,12 @@ private void deleteOldPacks(Collection oldPacks, prunePreserved(); long packExpireDate = getPackExpireDate(); - oldPackLoop: for (PackFile oldPack : oldPacks) { + oldPackLoop: for (Pack oldPack : oldPacks) { checkCancelled(); String oldName = oldPack.getPackName(); // check whether an old pack file is also among the list of new // pack files. Then we must not delete it. - for (PackFile newPack : newPacks) + for (Pack newPack : newPacks) if (oldName.equals(newPack.getPackName())) continue oldPackLoop; @@ -439,7 +438,7 @@ private void prunePack(String packName) { */ public void prunePacked() throws IOException { ObjectDirectory objdb = repo.getObjectDatabase(); - Collection packs = objdb.getPacks(); + Collection packs = objdb.getPacks(); File objects = repo.getObjectsDirectory(); String[] fanout = objects.list(); @@ -467,7 +466,7 @@ public void prunePacked() throws IOException { continue; } boolean found = false; - for (PackFile p : packs) { + for (Pack p : packs) { checkCancelled(); if (p.hasObject(id)) { found = true; @@ -789,8 +788,8 @@ public void packRefs() throws IOException { * reflog-entries or during writing to the packfiles * {@link java.io.IOException} occurs */ - public Collection repack() throws IOException { - Collection toBeDeleted = repo.getObjectDatabase().getPacks(); + public Collection repack() throws IOException { + Collection toBeDeleted = repo.getObjectDatabase().getPacks(); long time = System.currentTimeMillis(); Collection refsBefore = getAllRefs(); @@ -802,7 +801,6 @@ public Collection repack() throws IOException { Set txnHeads = new HashSet<>(); Set tagTargets = new HashSet<>(); Set indexObjects = listNonHEADIndexObjects(); - RefDatabase refdb = repo.getRefDatabase(); for (Ref ref : refsBefore) { checkCancelled(); @@ -814,8 +812,6 @@ public Collection repack() throws IOException { allHeads.add(ref.getObjectId()); } else if (isTag(ref)) { allTags.add(ref.getObjectId()); - } else if (RefTreeNames.isRefTree(refdb, ref.getName())) { - txnHeads.add(ref.getObjectId()); } else { nonHeads.add(ref.getObjectId()); } @@ -825,10 +821,10 @@ public Collection repack() throws IOException { } List excluded = new LinkedList<>(); - for (PackFile f : repo.getObjectDatabase().getPacks()) { + for (Pack p : repo.getObjectDatabase().getPacks()) { checkCancelled(); - if (f.shouldBeKept()) - excluded.add(f.getIndex()); + if (p.shouldBeKept()) + excluded.add(p.getIndex()); } // Don't exclude tags that are also branch tips @@ -846,8 +842,8 @@ public Collection repack() throws IOException { nonHeads.clear(); } - List ret = new ArrayList<>(2); - PackFile heads = null; + List ret = new ArrayList<>(2); + Pack heads = null; if (!allHeadsAndTags.isEmpty()) { heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, tagTargets, excluded); @@ -857,13 +853,13 @@ public Collection repack() throws IOException { } } if (!nonHeads.isEmpty()) { - PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, + Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, tagTargets, excluded); if (rest != null) ret.add(rest); } if (!txnHeads.isEmpty()) { - PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, + Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, null, excluded); if (txn != null) ret.add(txn); @@ -1133,7 +1129,7 @@ private Set listNonHEADIndexObjects() } } - private PackFile writePack(@NonNull Set want, + private Pack writePack(@NonNull Set want, @NonNull Set have, @NonNull Set tags, Set tagTargets, List excludeObjects) throws IOException { @@ -1360,13 +1356,13 @@ public String toString() { */ public RepoStatistics getStatistics() throws IOException { RepoStatistics ret = new RepoStatistics(); - Collection packs = repo.getObjectDatabase().getPacks(); - for (PackFile f : packs) { - ret.numberOfPackedObjects += f.getIndex().getObjectCount(); + Collection packs = repo.getObjectDatabase().getPacks(); + for (Pack p : packs) { + ret.numberOfPackedObjects += p.getIndex().getObjectCount(); ret.numberOfPackFiles++; - ret.sizeOfPackedObjects += f.getPackFile().length(); - if (f.getBitmapIndex() != null) - ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount(); + ret.sizeOfPackedObjects += p.getPackFile().length(); + if (p.getBitmapIndex() != null) + ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java index ee4bbc196..e2fbd7a0b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java @@ -30,12 +30,12 @@ class LargePackedWholeObject extends ObjectLoader { private final int headerLength; - private final PackFile pack; + private final Pack pack; private final FileObjectDatabase db; LargePackedWholeObject(int type, long size, long objectOffset, - int headerLength, PackFile pack, FileObjectDatabase db) { + int headerLength, Pack pack, FileObjectDatabase db) { this.type = type; this.size = size; this.objectOffset = objectOffset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index 9d04062e3..ae5bce698 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -25,31 +25,31 @@ class LocalCachedPack extends CachedPack { private final String[] packNames; - private PackFile[] packs; + private Pack[] packs; LocalCachedPack(ObjectDirectory odb, List packNames) { this.odb = odb; this.packNames = packNames.toArray(new String[0]); } - LocalCachedPack(List packs) { + LocalCachedPack(List packs) { odb = null; packNames = null; - this.packs = packs.toArray(new PackFile[0]); + this.packs = packs.toArray(new Pack[0]); } /** {@inheritDoc} */ @Override public long getObjectCount() throws IOException { long cnt = 0; - for (PackFile pack : getPacks()) + for (Pack pack : getPacks()) cnt += pack.getObjectCount(); return cnt; } void copyAsIs(PackOutputStream out, WindowCursor wc) throws IOException { - for (PackFile pack : getPacks()) + for (Pack pack : getPacks()) pack.copyPackAsIs(out, wc); } @@ -58,7 +58,7 @@ void copyAsIs(PackOutputStream out, WindowCursor wc) public boolean hasObject(ObjectToPack obj, StoredObjectRepresentation rep) { try { LocalObjectRepresentation local = (LocalObjectRepresentation) rep; - for (PackFile pack : getPacks()) { + for (Pack pack : getPacks()) { if (local.pack == pack) return true; } @@ -68,9 +68,9 @@ public boolean hasObject(ObjectToPack obj, StoredObjectRepresentation rep) { } } - private PackFile[] getPacks() throws FileNotFoundException { + private Pack[] getPacks() throws FileNotFoundException { if (packs == null) { - PackFile[] p = new PackFile[packNames.length]; + Pack[] p = new Pack[packNames.length]; for (int i = 0; i < packNames.length; i++) p[i] = getPackFile(packNames[i]); packs = p; @@ -78,8 +78,8 @@ private PackFile[] getPacks() throws FileNotFoundException { return packs; } - private PackFile getPackFile(String packName) throws FileNotFoundException { - for (PackFile pack : odb.getPacks()) { + private Pack getPackFile(String packName) throws FileNotFoundException { + for (Pack pack : odb.getPacks()) { if (packName.equals(pack.getPackName())) return pack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java index 3950dde4a..559718af3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java @@ -16,40 +16,40 @@ import org.eclipse.jgit.lib.ObjectId; class LocalObjectRepresentation extends StoredObjectRepresentation { - static LocalObjectRepresentation newWhole(PackFile f, long p, long length) { + static LocalObjectRepresentation newWhole(Pack pack, long offset, long length) { LocalObjectRepresentation r = new LocalObjectRepresentation() { @Override public int getFormat() { return PACK_WHOLE; } }; - r.pack = f; - r.offset = p; + r.pack = pack; + r.offset = offset; r.length = length; return r; } - static LocalObjectRepresentation newDelta(PackFile f, long p, long n, + static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, ObjectId base) { LocalObjectRepresentation r = new Delta(); - r.pack = f; - r.offset = p; - r.length = n; + r.pack = pack; + r.offset = offset; + r.length = length; r.baseId = base; return r; } - static LocalObjectRepresentation newDelta(PackFile f, long p, long n, + static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, long base) { LocalObjectRepresentation r = new Delta(); - r.pack = f; - r.offset = p; - r.length = n; + r.pack = pack; + r.offset = offset; + r.length = length; r.baseOffset = base; return r; } - PackFile pack; + Pack pack; long offset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java index 4a0ac1fd8..ac6cd212d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java @@ -17,7 +17,7 @@ /** {@link ObjectToPack} for {@link ObjectDirectory}. */ class LocalObjectToPack extends ObjectToPack { /** Pack to reuse compressed data from, otherwise null. */ - PackFile pack; + Pack pack; /** Offset of the object's header in {@link #pack}. */ long offset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java new file mode 100644 index 000000000..e7cb285c3 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2009, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.StandardCopyOption; +import java.util.Set; + +import org.eclipse.jgit.internal.storage.file.FileObjectDatabase.InsertLooseObjectResult; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Traditional file system based loose objects handler. + *

    + * This is the loose object representation for a Git object database, where + * objects are stored loose by hashing them into directories by their + * {@link org.eclipse.jgit.lib.ObjectId}. + */ +class LooseObjects { + private static final Logger LOG = LoggerFactory + .getLogger(LooseObjects.class); + + private final File directory; + + private final UnpackedObjectCache unpackedObjectCache; + + /** + * Initialize a reference to an on-disk object directory. + * + * @param dir + * the location of the objects directory. + */ + LooseObjects(File dir) { + directory = dir; + unpackedObjectCache = new UnpackedObjectCache(); + } + + /** + * Getter for the field directory. + * + * @return the location of the objects directory. + */ + File getDirectory() { + return directory; + } + + void create() throws IOException { + FileUtils.mkdirs(directory); + } + + void close() { + unpackedObjectCache.clear(); + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return "LooseObjects[" + directory + "]"; //$NON-NLS-1$ //$NON-NLS-2$ + } + + boolean hasCached(AnyObjectId id) { + return unpackedObjectCache.isUnpacked(id); + } + + /** + * Does the requested object exist as a loose object? + * + * @param objectId + * identity of the object to test for existence of. + * @return {@code true} if the specified object is stored as a loose object. + */ + boolean has(AnyObjectId objectId) { + return fileFor(objectId).exists(); + } + + /** + * Find objects matching the prefix abbreviation. + * + * @param matches + * set to add any located ObjectIds to. This is an output + * parameter. + * @param id + * prefix to search for. + * @param matchLimit + * maximum number of results to return. At most this many + * ObjectIds should be added to matches before returning. + * @return {@code true} if the matches were exhausted before reaching + * {@code maxLimit}. + */ + boolean resolve(Set matches, AbbreviatedObjectId id, + int matchLimit) { + String fanOut = id.name().substring(0, 2); + String[] entries = new File(directory, fanOut).list(); + if (entries != null) { + for (String e : entries) { + if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) { + continue; + } + try { + ObjectId entId = ObjectId.fromString(fanOut + e); + if (id.prefixCompare(entId) == 0) { + matches.add(entId); + } + } catch (IllegalArgumentException notId) { + continue; + } + if (matches.size() > matchLimit) { + return false; + } + } + } + return true; + } + + ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { + File path = fileFor(id); + try (FileInputStream in = new FileInputStream(path)) { + unpackedObjectCache.add(id); + return UnpackedObject.open(in, path, id, curs); + } catch (FileNotFoundException noFile) { + if (path.exists()) { + throw noFile; + } + unpackedObjectCache.remove(id); + return null; + } + } + + long getSize(WindowCursor curs, AnyObjectId id) throws IOException { + File f = fileFor(id); + try (FileInputStream in = new FileInputStream(f)) { + unpackedObjectCache.add(id); + return UnpackedObject.getSize(in, id, curs); + } catch (FileNotFoundException noFile) { + if (f.exists()) { + throw noFile; + } + unpackedObjectCache.remove(id); + return -1; + } + } + + InsertLooseObjectResult insert(File tmp, ObjectId id) throws IOException { + final File dst = fileFor(id); + if (dst.exists()) { + // We want to be extra careful and avoid replacing an object + // that already exists. We can't be sure renameTo() would + // fail on all platforms if dst exists, so we check first. + // + FileUtils.delete(tmp, FileUtils.RETRY); + return InsertLooseObjectResult.EXISTS_LOOSE; + } + + try { + return tryMove(tmp, dst, id); + } catch (NoSuchFileException e) { + // It's possible the directory doesn't exist yet as the object + // directories are always lazily created. Note that we try the + // rename/move first as the directory likely does exist. + // + // Create the directory. + // + FileUtils.mkdir(dst.getParentFile(), true); + } catch (IOException e) { + // Any other IO error is considered a failure. + // + LOG.error(e.getMessage(), e); + FileUtils.delete(tmp, FileUtils.RETRY); + return InsertLooseObjectResult.FAILURE; + } + + try { + return tryMove(tmp, dst, id); + } catch (IOException e) { + // The object failed to be renamed into its proper location and + // it doesn't exist in the repository either. We really don't + // know what went wrong, so fail. + // + LOG.error(e.getMessage(), e); + FileUtils.delete(tmp, FileUtils.RETRY); + return InsertLooseObjectResult.FAILURE; + } + } + + private InsertLooseObjectResult tryMove(File tmp, File dst, ObjectId id) + throws IOException { + Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst), + StandardCopyOption.ATOMIC_MOVE); + dst.setReadOnly(); + unpackedObjectCache.add(id); + return InsertLooseObjectResult.INSERTED; + } + + /** + * Compute the location of a loose object file. + * + * @param objectId + * identity of the object to get the File location for. + * @return {@link java.io.File} location of the specified loose object. + */ + File fileFor(AnyObjectId objectId) { + String n = objectId.name(); + String d = n.substring(0, 2); + String f = n.substring(2); + return new File(new File(getDirectory(), d), f); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index d32182864..e71a96060 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -16,28 +16,19 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.PackInvalidException; -import org.eclipse.jgit.errors.PackMismatchException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -45,7 +36,6 @@ import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectId; @@ -54,8 +44,6 @@ import org.eclipse.jgit.lib.RepositoryCache.FileKey; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Traditional file system based {@link org.eclipse.jgit.lib.ObjectDatabase}. @@ -63,7 +51,7 @@ * This is the classical object database representation for a Git repository, * where objects are stored loose by hashing them into directories by their * {@link org.eclipse.jgit.lib.ObjectId}, or are stored in compressed containers - * known as {@link org.eclipse.jgit.internal.storage.file.PackFile}s. + * known as {@link org.eclipse.jgit.internal.storage.file.Pack}s. *

    * Optionally an object database can reference one or more alternates; other * ObjectDatabase instances that are searched in addition to the current @@ -76,12 +64,6 @@ * considered. */ public class ObjectDirectory extends FileObjectDatabase { - private static final Logger LOG = LoggerFactory - .getLogger(ObjectDirectory.class); - - private static final PackList NO_PACKS = new PackList( - FileSnapshot.DIRTY, new PackFile[0]); - /** Maximum number of candidates offered as resolutions of abbreviation. */ private static final int RESOLVE_ABBREV_LIMIT = 256; @@ -93,7 +75,9 @@ public class ObjectDirectory extends FileObjectDatabase { private final File infoDirectory; - private final File packDirectory; + private final LooseObjects loose; + + private final PackDirectory packed; private final File preservedDirectory; @@ -103,16 +87,12 @@ public class ObjectDirectory extends FileObjectDatabase { private final AtomicReference alternates; - private final UnpackedObjectCache unpackedObjectCache; - private final File shallowFile; private FileSnapshot shallowFileSnapshot = FileSnapshot.DIRTY; private Set shallowCommitsIds; - final AtomicReference packList; - /** * Initialize a reference to an on-disk object directory. * @@ -136,11 +116,11 @@ public ObjectDirectory(final Config cfg, final File dir, config = cfg; objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ - packDirectory = new File(objects, "pack"); //$NON-NLS-1$ + File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); - packList = new AtomicReference<>(NO_PACKS); - unpackedObjectCache = new UnpackedObjectCache(); + loose = new LooseObjects(objects); + packed = new PackDirectory(config, packDirectory); this.fs = fs; this.shallowFile = shallowFile; @@ -158,7 +138,7 @@ public ObjectDirectory(final Config cfg, final File dir, /** {@inheritDoc} */ @Override public final File getDirectory() { - return objects; + return loose.getDirectory(); } /** @@ -167,7 +147,7 @@ public final File getDirectory() { * @return the location of the pack directory. */ public final File getPackDirectory() { - return packDirectory; + return packed.getDirectory(); } /** @@ -188,9 +168,9 @@ public boolean exists() { /** {@inheritDoc} */ @Override public void create() throws IOException { - FileUtils.mkdirs(objects); + loose.create(); FileUtils.mkdir(infoDirectory); - FileUtils.mkdir(packDirectory); + packed.create(); } /** {@inheritDoc} */ @@ -212,13 +192,9 @@ public PackInserter newPackInserter() { /** {@inheritDoc} */ @Override public void close() { - unpackedObjectCache.clear(); + loose.close(); - final PackList packs = packList.get(); - if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { - for (PackFile p : packs.packs) - p.close(); - } + packed.close(); // Fully close all loaded alternates and clear the alternate list. AlternateHandle[] alt = alternates.get(); @@ -230,12 +206,8 @@ public void close() { /** {@inheritDoc} */ @Override - public Collection getPacks() { - PackList list = packList.get(); - if (list == NO_PACKS) - list = scanPacks(list); - PackFile[] packs = list.packs; - return Collections.unmodifiableCollection(Arrays.asList(packs)); + public Collection getPacks() { + return packed.getPacks(); } /** @@ -244,7 +216,7 @@ public Collection getPacks() { * Add a single existing pack to the list of available pack files. */ @Override - public PackFile openPack(File pack) + public Pack openPack(File pack) throws IOException { final String p = pack.getName(); if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ @@ -263,8 +235,8 @@ public PackFile openPack(File pack) } } - PackFile res = new PackFile(pack, extensions); - insertPack(res); + Pack res = new Pack(pack, extensions); + packed.insert(res); return res; } @@ -277,7 +249,7 @@ public String toString() { /** {@inheritDoc} */ @Override public boolean has(AnyObjectId objectId) { - return unpackedObjectCache.isUnpacked(objectId) + return loose.hasCached(objectId) || hasPackedInSelfOrAlternate(objectId, null) || hasLooseInSelfOrAlternate(objectId, null); } @@ -300,7 +272,7 @@ private boolean hasPackedInSelfOrAlternate(AnyObjectId objectId, private boolean hasLooseInSelfOrAlternate(AnyObjectId objectId, Set skips) { - if (fileFor(objectId).exists()) { + if (loose.has(objectId)) { return true; } skips = addMe(skips); @@ -315,25 +287,7 @@ private boolean hasLooseInSelfOrAlternate(AnyObjectId objectId, } boolean hasPackedObject(AnyObjectId objectId) { - PackList pList; - do { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - if (p.hasObject(objectId)) - return true; - } catch (IOException e) { - // The hasObject call should have only touched the index, - // so any failure here indicates the index is unreadable - // by this process, and the pack is likewise not readable. - LOG.warn(MessageFormat.format( - JGitText.get().unableToReadPackfile, - p.getPackFile().getAbsolutePath()), e); - removePack(p); - } - } - } while (searchPacksAgain(pList)); - return false; + return packed.has(objectId); } @Override @@ -345,41 +299,11 @@ void resolve(Set matches, AbbreviatedObjectId id) private void resolve(Set matches, AbbreviatedObjectId id, Set skips) throws IOException { - // Go through the packs once. If we didn't find any resolutions - // scan for new packs and check once more. - int oldSize = matches.size(); - PackList pList; - do { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - p.resolve(matches, id, RESOLVE_ABBREV_LIMIT); - p.resetTransientErrorCount(); - } catch (IOException e) { - handlePackError(e, p); - } - if (matches.size() > RESOLVE_ABBREV_LIMIT) - return; - } - } while (matches.size() == oldSize && searchPacksAgain(pList)); + if (!packed.resolve(matches, id, RESOLVE_ABBREV_LIMIT)) + return; - String fanOut = id.name().substring(0, 2); - String[] entries = new File(getDirectory(), fanOut).list(); - if (entries != null) { - for (String e : entries) { - if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) - continue; - try { - ObjectId entId = ObjectId.fromString(fanOut + e); - if (id.prefixCompare(entId) == 0) - matches.add(entId); - } catch (IllegalArgumentException notId) { - continue; - } - if (matches.size() > RESOLVE_ABBREV_LIMIT) - return; - } - } + if (!loose.resolve(matches, id, RESOLVE_ABBREV_LIMIT)) + return; skips = addMe(skips); for (AlternateHandle alt : myAlternates()) { @@ -395,7 +319,7 @@ private void resolve(Set matches, AbbreviatedObjectId id, @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { - if (unpackedObjectCache.isUnpacked(objectId)) { + if (loose.hasCached(objectId)) { ObjectLoader ldr = openLooseObject(curs, objectId); if (ldr != null) { return ldr; @@ -446,51 +370,20 @@ private ObjectLoader openLooseFromSelfOrAlternate(WindowCursor curs, } ObjectLoader openPackedObject(WindowCursor curs, AnyObjectId objectId) { - PackList pList; - do { - SEARCH: for (;;) { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - ObjectLoader ldr = p.get(curs, objectId); - p.resetTransientErrorCount(); - if (ldr != null) - return ldr; - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - if (searchPacksAgain(pList)) - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } - } while (searchPacksAgain(pList)); - return null; + return packed.open(curs, objectId); } @Override ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id) throws IOException { - File path = fileFor(id); - try (FileInputStream in = new FileInputStream(path)) { - unpackedObjectCache.add(id); - return UnpackedObject.open(in, path, id, curs); - } catch (FileNotFoundException noFile) { - if (path.exists()) { - throw noFile; - } - unpackedObjectCache.remove(id); - return null; - } + return loose.open(curs, id); } @Override long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { - if (unpackedObjectCache.isUnpacked(id)) { - long len = getLooseObjectSize(curs, id); + if (loose.hasCached(id)) { + long len = loose.getSize(curs, id); if (0 <= len) { return len; } @@ -504,7 +397,7 @@ long getObjectSize(WindowCursor curs, AnyObjectId id) private long getPackedSizeFromSelfOrAlternate(WindowCursor curs, AnyObjectId id, Set skips) { - long len = getPackedObjectSize(curs, id); + long len = packed.getSize(curs, id); if (0 <= len) { return len; } @@ -522,7 +415,7 @@ private long getPackedSizeFromSelfOrAlternate(WindowCursor curs, private long getLooseSizeFromSelfOrAlternate(WindowCursor curs, AnyObjectId id, Set skips) throws IOException { - long len = getLooseObjectSize(curs, id); + long len = loose.getSize(curs, id); if (0 <= len) { return len; } @@ -538,46 +431,6 @@ private long getLooseSizeFromSelfOrAlternate(WindowCursor curs, return -1; } - private long getPackedObjectSize(WindowCursor curs, AnyObjectId id) { - PackList pList; - do { - SEARCH: for (;;) { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - long len = p.getObjectSize(curs, id); - p.resetTransientErrorCount(); - if (0 <= len) - return len; - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - if (searchPacksAgain(pList)) - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } - } while (searchPacksAgain(pList)); - return -1; - } - - private long getLooseObjectSize(WindowCursor curs, AnyObjectId id) - throws IOException { - File f = fileFor(id); - try (FileInputStream in = new FileInputStream(f)) { - unpackedObjectCache.add(id); - return UnpackedObject.getSize(in, id, curs); - } catch (FileNotFoundException noFile) { - if (f.exists()) { - throw noFile; - } - unpackedObjectCache.remove(id); - return -1; - } - } - @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { @@ -586,25 +439,7 @@ void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs, Set skips) throws IOException { - PackList pList = packList.get(); - SEARCH: for (;;) { - for (PackFile p : pList.packs) { - try { - LocalObjectRepresentation rep = p.representation(curs, otp); - p.resetTransientErrorCount(); - if (rep != null) - packer.select(otp, rep); - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - // - pList = scanPacks(pList); - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } + packed.selectRepresentation(packer, otp, curs); skips = addMe(skips); for (AlternateHandle h : myAlternates()) { @@ -614,60 +449,12 @@ private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, } } - private void handlePackError(IOException e, PackFile p) { - String warnTmpl = null; - int transientErrorCount = 0; - String errTmpl = JGitText.get().exceptionWhileReadingPack; - if ((e instanceof CorruptObjectException) - || (e instanceof PackInvalidException)) { - warnTmpl = JGitText.get().corruptPack; - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - // Assume the pack is corrupted, and remove it from the list. - removePack(p); - } else if (e instanceof FileNotFoundException) { - if (p.getPackFile().exists()) { - errTmpl = JGitText.get().packInaccessible; - transientErrorCount = p.incrementTransientErrorCount(); - } else { - warnTmpl = JGitText.get().packWasDeleted; - removePack(p); - } - } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { - warnTmpl = JGitText.get().packHandleIsStale; - removePack(p); - } else { - transientErrorCount = p.incrementTransientErrorCount(); - } - if (warnTmpl != null) { - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - } else { - if (doLogExponentialBackoff(transientErrorCount)) { - // Don't remove the pack from the list, as the error may be - // transient. - LOG.error(MessageFormat.format(errTmpl, - p.getPackFile().getAbsolutePath(), - Integer.valueOf(transientErrorCount)), e); - } - } - } - - /** - * @param n - * count of consecutive failures - * @return @{code true} if i is a power of 2 - */ - private boolean doLogExponentialBackoff(int n) { - return (n & (n - 1)) == 0; - } - @Override InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { // If the object is already in the repository, remove temporary file. // - if (unpackedObjectCache.isUnpacked(id)) { + if (loose.hasCached(id)) { FileUtils.delete(tmp, FileUtils.RETRY); return InsertLooseObjectResult.EXISTS_LOOSE; } @@ -675,71 +462,7 @@ InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, FileUtils.delete(tmp, FileUtils.RETRY); return InsertLooseObjectResult.EXISTS_PACKED; } - - final File dst = fileFor(id); - if (dst.exists()) { - // We want to be extra careful and avoid replacing an object - // that already exists. We can't be sure renameTo() would - // fail on all platforms if dst exists, so we check first. - // - FileUtils.delete(tmp, FileUtils.RETRY); - return InsertLooseObjectResult.EXISTS_LOOSE; - } - - try { - return tryMove(tmp, dst, id); - } catch (NoSuchFileException e) { - // It's possible the directory doesn't exist yet as the object - // directories are always lazily created. Note that we try the - // rename/move first as the directory likely does exist. - // - // Create the directory. - // - FileUtils.mkdir(dst.getParentFile(), true); - } catch (IOException e) { - // Any other IO error is considered a failure. - // - LOG.error(e.getMessage(), e); - FileUtils.delete(tmp, FileUtils.RETRY); - return InsertLooseObjectResult.FAILURE; - } - - try { - return tryMove(tmp, dst, id); - } catch (IOException e) { - // The object failed to be renamed into its proper location and - // it doesn't exist in the repository either. We really don't - // know what went wrong, so fail. - // - LOG.error(e.getMessage(), e); - FileUtils.delete(tmp, FileUtils.RETRY); - return InsertLooseObjectResult.FAILURE; - } - } - - private InsertLooseObjectResult tryMove(File tmp, File dst, - ObjectId id) - throws IOException { - Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst), - StandardCopyOption.ATOMIC_MOVE); - dst.setReadOnly(); - unpackedObjectCache.add(id); - return InsertLooseObjectResult.INSERTED; - } - - boolean searchPacksAgain(PackList old) { - // Whether to trust the pack folder's modification time. If set - // to false we will always scan the .git/objects/pack folder to - // check for new pack files. If set to true (default) we use the - // lastmodified attribute of the folder and assume that no new - // pack files can be in this folder if his modification time has - // not changed. - boolean trustFolderStat = config.getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - - return ((!trustFolderStat) || old.snapshot.isModified(packDirectory)) - && old != scanPacks(old); + return loose.insert(tmp, id); } @Override @@ -780,182 +503,13 @@ Set getShallowCommits() throws IOException { return shallowCommitsIds; } - private void insertPack(PackFile pf) { - PackList o, n; - do { - o = packList.get(); - - // If the pack in question is already present in the list - // (picked up by a concurrent thread that did a scan?) we - // do not want to insert it a second time. - // - final PackFile[] oldList = o.packs; - final String name = pf.getPackFile().getName(); - for (PackFile p : oldList) { - if (name.equals(p.getPackFile().getName())) - return; - } - - final PackFile[] newList = new PackFile[1 + oldList.length]; - newList[0] = pf; - System.arraycopy(oldList, 0, newList, 1, oldList.length); - n = new PackList(o.snapshot, newList); - } while (!packList.compareAndSet(o, n)); - } - - private void removePack(PackFile deadPack) { - PackList o, n; - do { - o = packList.get(); - - final PackFile[] oldList = o.packs; - final int j = indexOf(oldList, deadPack); - if (j < 0) - break; - - final PackFile[] newList = new PackFile[oldList.length - 1]; - System.arraycopy(oldList, 0, newList, 0, j); - System.arraycopy(oldList, j + 1, newList, j, newList.length - j); - n = new PackList(o.snapshot, newList); - } while (!packList.compareAndSet(o, n)); - deadPack.close(); - } - - private static int indexOf(PackFile[] list, PackFile pack) { - for (int i = 0; i < list.length; i++) { - if (list[i] == pack) - return i; - } - return -1; - } - - private PackList scanPacks(PackList original) { - synchronized (packList) { - PackList o, n; - do { - o = packList.get(); - if (o != original) { - // Another thread did the scan for us, while we - // were blocked on the monitor above. - // - return o; - } - n = scanPacksImpl(o); - if (n == o) - return n; - } while (!packList.compareAndSet(o, n)); - return n; - } - } - - private PackList scanPacksImpl(PackList old) { - final Map forReuse = reuseMap(old); - final FileSnapshot snapshot = FileSnapshot.save(packDirectory); - final Set names = listPackDirectory(); - final List list = new ArrayList<>(names.size() >> 2); - boolean foundNew = false; - for (String indexName : names) { - // Must match "pack-[0-9a-f]{40}.idx" to be an index. - // - if (indexName.length() != 49 || !indexName.endsWith(".idx")) //$NON-NLS-1$ - continue; - - final String base = indexName.substring(0, indexName.length() - 3); - int extensions = 0; - for (PackExt ext : PackExt.values()) { - if (names.contains(base + ext.getExtension())) - extensions |= ext.getBit(); - } - - if ((extensions & PACK.getBit()) == 0) { - // Sometimes C Git's HTTP fetch transport leaves a - // .idx file behind and does not download the .pack. - // We have to skip over such useless indexes. - // - continue; - } - - final String packName = base + PACK.getExtension(); - final File packFile = new File(packDirectory, packName); - final PackFile oldPack = forReuse.get(packName); - if (oldPack != null - && !oldPack.getFileSnapshot().isModified(packFile)) { - forReuse.remove(packName); - list.add(oldPack); - continue; - } - - list.add(new PackFile(packFile, extensions)); - foundNew = true; - } - - // If we did not discover any new files, the modification time was not - // changed, and we did not remove any files, then the set of files is - // the same as the set we were given. Instead of building a new object - // return the same collection. - // - if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) { - old.snapshot.setClean(snapshot); - return old; - } - - for (PackFile p : forReuse.values()) { - p.close(); - } - - if (list.isEmpty()) - return new PackList(snapshot, NO_PACKS.packs); - - final PackFile[] r = list.toArray(new PackFile[0]); - Arrays.sort(r, PackFile.SORT); - return new PackList(snapshot, r); - } - - private static Map reuseMap(PackList old) { - final Map forReuse = new HashMap<>(); - for (PackFile p : old.packs) { - if (p.invalid()) { - // The pack instance is corrupted, and cannot be safely used - // again. Do not include it in our reuse map. - // - p.close(); - continue; - } - - final PackFile prior = forReuse.put(p.getPackFile().getName(), p); - if (prior != null) { - // This should never occur. It should be impossible for us - // to have two pack files with the same name, as all of them - // came out of the same directory. If it does, we promised to - // close any PackFiles we did not reuse, so close the second, - // readers are likely to be actively using the first. - // - forReuse.put(prior.getPackFile().getName(), prior); - p.close(); - } - } - return forReuse; - } - - private Set listPackDirectory() { - final String[] nameList = packDirectory.list(); - if (nameList == null) - return Collections.emptySet(); - final Set nameSet = new HashSet<>(nameList.length << 1); - for (String name : nameList) { - if (name.startsWith("pack-")) //$NON-NLS-1$ - nameSet.add(name); - } - return nameSet; - } - void closeAllPackHandles(File packFile) { // if the packfile already exists (because we are rewriting a // packfile for the same set of objects maybe with different // PackConfig) then make sure we get rid of all handles on the file. // Windows will not allow for rename otherwise. if (packFile.exists()) { - for (PackFile p : getPacks()) { + for (Pack p : packed.getPacks()) { if (packFile.getPath().equals(p.getPackFile().getPath())) { p.close(); break; @@ -1025,29 +579,11 @@ private AlternateHandle openAlternate(File objdir) throws IOException { } /** - * {@inheritDoc} - *

    * Compute the location of a loose object file. */ @Override public File fileFor(AnyObjectId objectId) { - String n = objectId.name(); - String d = n.substring(0, 2); - String f = n.substring(2); - return new File(new File(getDirectory(), d), f); - } - - static final class PackList { - /** State just before reading the pack directory. */ - final FileSnapshot snapshot; - - /** All known packs, sorted by {@link PackFile#SORT}. */ - final PackFile[] packs; - - PackList(FileSnapshot monitor, PackFile[] packs) { - this.snapshot = monitor; - this.packs = packs; - } + return loose.fileFor(objectId); } static class AlternateHandle { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index e27518690..04d2ff8ab 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -88,7 +88,7 @@ public class ObjectDirectoryPackParser extends PackParser { private Deflater def; /** The pack that was created, if parsing was successful. */ - private PackFile newPack; + private Pack newPack; private PackConfig pconfig; @@ -129,14 +129,14 @@ public void setKeepEmpty(boolean empty) { } /** - * Get the imported {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * Get the imported {@link org.eclipse.jgit.internal.storage.file.Pack}. *

    * This method is supplied only to support testing; applications shouldn't * be using it directly to access the imported data. * * @return the imported PackFile, if parsing was successful. */ - public PackFile getPackFile() { + public Pack getPack() { return newPack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java similarity index 98% rename from org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 254c02023..d928633a7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -69,13 +69,13 @@ * delta packed format yielding high compression of lots of object where some * objects are similar. */ -public class PackFile implements Iterable { - private static final Logger LOG = LoggerFactory.getLogger(PackFile.class); +public class Pack implements Iterable { + private static final Logger LOG = LoggerFactory.getLogger(Pack.class); /** * Sorts PackFiles to be most recently created to least recently created. */ - public static final Comparator SORT = (a, b) -> b.packLastModified + public static final Comparator SORT = (a, b) -> b.packLastModified .compareTo(a.packLastModified); private final File packFile; @@ -136,7 +136,7 @@ public class PackFile implements Iterable { * @param extensions * additional pack file extensions with the same base as the pack */ - public PackFile(File packFile, int extensions) { + public Pack(File packFile, int extensions) { this.packFile = packFile; this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = fileSnapshot.lastModifiedInstant(); @@ -650,6 +650,7 @@ synchronized boolean endWindowCache() { private void doOpen() throws IOException { if (invalid) { + openFail(true, invalidatingCause); throw new PackInvalidException(packFile, invalidatingCause); } try { @@ -1200,7 +1201,7 @@ private boolean hasExt(PackExt ext) { @SuppressWarnings("nls") @Override public String toString() { - return "PackFile [packFileName=" + packFile.getName() + ", length=" + return "Pack [packFileName=" + packFile.getName() + ", length=" + packFile.length() + ", packChecksum=" + ObjectId.fromRaw(packChecksum).name() + "]"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java new file mode 100644 index 000000000..b2ba36bf9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -0,0 +1,516 @@ +/* + * Copyright (C) 2009, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackInvalidException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Traditional file system packed objects directory handler. + *

    + * This is the {@link org.eclipse.jgit.internal.storage.file.Pack}s object + * representation for a Git object database, where objects are stored in + * compressed containers known as + * {@link org.eclipse.jgit.internal.storage.file.Pack}s. + */ +class PackDirectory { + private final static Logger LOG = LoggerFactory + .getLogger(PackDirectory.class); + + private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY, + new Pack[0]); + + private final Config config; + + private final File directory; + + private final AtomicReference packList; + + /** + * Initialize a reference to an on-disk 'pack' directory. + * + * @param config + * configuration this directory consults for write settings. + * @param directory + * the location of the {@code pack} directory. + */ + PackDirectory(Config config, File directory) { + this.config = config; + this.directory = directory; + packList = new AtomicReference<>(NO_PACKS); + } + + /** + * Getter for the field {@code directory}. + * + * @return the location of the {@code pack} directory. + */ + File getDirectory() { + return directory; + } + + void create() throws IOException { + FileUtils.mkdir(directory); + } + + void close() { + PackList packs = packList.get(); + if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { + for (Pack p : packs.packs) { + p.close(); + } + } + } + + Collection getPacks() { + PackList list = packList.get(); + if (list == NO_PACKS) { + list = scanPacks(list); + } + Pack[] packs = list.packs; + return Collections.unmodifiableCollection(Arrays.asList(packs)); + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return "PackDirectory[" + getDirectory() + "]"; //$NON-NLS-1$ //$NON-NLS-2$ + } + + /** + * Does the requested object exist in this PackDirectory? + * + * @param objectId + * identity of the object to test for existence of. + * @return true if the specified object is stored in this PackDirectory. + */ + boolean has(AnyObjectId objectId) { + PackList pList; + do { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + if (p.hasObject(objectId)) { + return true; + } + } catch (IOException e) { + // The hasObject call should have only touched the index, + // so any failure here indicates the index is unreadable + // by this process, and the pack is likewise not readable. + LOG.warn(MessageFormat.format( + JGitText.get().unableToReadPackfile, + p.getPackFile().getAbsolutePath()), e); + remove(p); + } + } + } while (searchPacksAgain(pList)); + return false; + } + + /** + * Find objects matching the prefix abbreviation. + * + * @param matches + * set to add any located ObjectIds to. This is an output + * parameter. + * @param id + * prefix to search for. + * @param matchLimit + * maximum number of results to return. At most this many + * ObjectIds should be added to matches before returning. + * @return {@code true} if the matches were exhausted before reaching + * {@code maxLimit}. + */ + boolean resolve(Set matches, AbbreviatedObjectId id, + int matchLimit) { + // Go through the packs once. If we didn't find any resolutions + // scan for new packs and check once more. + int oldSize = matches.size(); + PackList pList; + do { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + p.resolve(matches, id, matchLimit); + p.resetTransientErrorCount(); + } catch (IOException e) { + handlePackError(e, p); + } + if (matches.size() > matchLimit) { + return false; + } + } + } while (matches.size() == oldSize && searchPacksAgain(pList)); + return true; + } + + ObjectLoader open(WindowCursor curs, AnyObjectId objectId) { + PackList pList; + do { + SEARCH: for (;;) { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + ObjectLoader ldr = p.get(curs, objectId); + p.resetTransientErrorCount(); + if (ldr != null) + return ldr; + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + if (searchPacksAgain(pList)) { + continue SEARCH; + } + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } while (searchPacksAgain(pList)); + return null; + } + + long getSize(WindowCursor curs, AnyObjectId id) { + PackList pList; + do { + SEARCH: for (;;) { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + long len = p.getObjectSize(curs, id); + p.resetTransientErrorCount(); + if (0 <= len) { + return len; + } + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + if (searchPacksAgain(pList)) { + continue SEARCH; + } + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } while (searchPacksAgain(pList)); + return -1; + } + + void selectRepresentation(PackWriter packer, ObjectToPack otp, + WindowCursor curs) { + PackList pList = packList.get(); + SEARCH: for (;;) { + for (Pack p : pList.packs) { + try { + LocalObjectRepresentation rep = p.representation(curs, otp); + p.resetTransientErrorCount(); + if (rep != null) { + packer.select(otp, rep); + } + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + // + pList = scanPacks(pList); + continue SEARCH; + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } + + private void handlePackError(IOException e, Pack p) { + String warnTmpl = null; + int transientErrorCount = 0; + String errTmpl = JGitText.get().exceptionWhileReadingPack; + if ((e instanceof CorruptObjectException) + || (e instanceof PackInvalidException)) { + warnTmpl = JGitText.get().corruptPack; + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); + // Assume the pack is corrupted, and remove it from the list. + remove(p); + } else if (e instanceof FileNotFoundException) { + if (p.getPackFile().exists()) { + errTmpl = JGitText.get().packInaccessible; + transientErrorCount = p.incrementTransientErrorCount(); + } else { + warnTmpl = JGitText.get().packWasDeleted; + remove(p); + } + } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { + warnTmpl = JGitText.get().packHandleIsStale; + remove(p); + } else { + transientErrorCount = p.incrementTransientErrorCount(); + } + if (warnTmpl != null) { + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); + } else { + if (doLogExponentialBackoff(transientErrorCount)) { + // Don't remove the pack from the list, as the error may be + // transient. + LOG.error(MessageFormat.format(errTmpl, + p.getPackFile().getAbsolutePath(), + Integer.valueOf(transientErrorCount)), e); + } + } + } + + /** + * @param n + * count of consecutive failures + * @return @{code true} if i is a power of 2 + */ + private boolean doLogExponentialBackoff(int n) { + return (n & (n - 1)) == 0; + } + + boolean searchPacksAgain(PackList old) { + // Whether to trust the pack folder's modification time. If set + // to false we will always scan the .git/objects/pack folder to + // check for new pack files. If set to true (default) we use the + // lastmodified attribute of the folder and assume that no new + // pack files can be in this folder if his modification time has + // not changed. + boolean trustFolderStat = config.getBoolean( + ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + + return ((!trustFolderStat) || old.snapshot.isModified(directory)) + && old != scanPacks(old); + } + + void insert(Pack pack) { + PackList o, n; + do { + o = packList.get(); + + // If the pack in question is already present in the list + // (picked up by a concurrent thread that did a scan?) we + // do not want to insert it a second time. + // + final Pack[] oldList = o.packs; + final String name = pack.getPackFile().getName(); + for (Pack p : oldList) { + if (name.equals(p.getPackFile().getName())) { + return; + } + } + + final Pack[] newList = new Pack[1 + oldList.length]; + newList[0] = pack; + System.arraycopy(oldList, 0, newList, 1, oldList.length); + n = new PackList(o.snapshot, newList); + } while (!packList.compareAndSet(o, n)); + } + + private void remove(Pack deadPack) { + PackList o, n; + do { + o = packList.get(); + + final Pack[] oldList = o.packs; + final int j = indexOf(oldList, deadPack); + if (j < 0) { + break; + } + + final Pack[] newList = new Pack[oldList.length - 1]; + System.arraycopy(oldList, 0, newList, 0, j); + System.arraycopy(oldList, j + 1, newList, j, newList.length - j); + n = new PackList(o.snapshot, newList); + } while (!packList.compareAndSet(o, n)); + deadPack.close(); + } + + private static int indexOf(Pack[] list, Pack pack) { + for (int i = 0; i < list.length; i++) { + if (list[i] == pack) { + return i; + } + } + return -1; + } + + private PackList scanPacks(PackList original) { + synchronized (packList) { + PackList o, n; + do { + o = packList.get(); + if (o != original) { + // Another thread did the scan for us, while we + // were blocked on the monitor above. + // + return o; + } + n = scanPacksImpl(o); + if (n == o) { + return n; + } + } while (!packList.compareAndSet(o, n)); + return n; + } + } + + private PackList scanPacksImpl(PackList old) { + final Map forReuse = reuseMap(old); + final FileSnapshot snapshot = FileSnapshot.save(directory); + final Set names = listPackDirectory(); + final List list = new ArrayList<>(names.size() >> 2); + boolean foundNew = false; + for (String indexName : names) { + // Must match "pack-[0-9a-f]{40}.idx" to be an index. + // + if (indexName.length() != 49 || !indexName.endsWith(".idx")) { //$NON-NLS-1$ + continue; + } + + final String base = indexName.substring(0, indexName.length() - 3); + int extensions = 0; + for (PackExt ext : PackExt.values()) { + if (names.contains(base + ext.getExtension())) { + extensions |= ext.getBit(); + } + } + + if ((extensions & PACK.getBit()) == 0) { + // Sometimes C Git's HTTP fetch transport leaves a + // .idx file behind and does not download the .pack. + // We have to skip over such useless indexes. + // + continue; + } + + final String packName = base + PACK.getExtension(); + final File packFile = new File(directory, packName); + final Pack oldPack = forReuse.get(packName); + if (oldPack != null + && !oldPack.getFileSnapshot().isModified(packFile)) { + forReuse.remove(packName); + list.add(oldPack); + continue; + } + + list.add(new Pack(packFile, extensions)); + foundNew = true; + } + + // If we did not discover any new files, the modification time was not + // changed, and we did not remove any files, then the set of files is + // the same as the set we were given. Instead of building a new object + // return the same collection. + // + if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) { + old.snapshot.setClean(snapshot); + return old; + } + + for (Pack p : forReuse.values()) { + p.close(); + } + + if (list.isEmpty()) { + return new PackList(snapshot, NO_PACKS.packs); + } + + final Pack[] r = list.toArray(new Pack[0]); + Arrays.sort(r, Pack.SORT); + return new PackList(snapshot, r); + } + + private static Map reuseMap(PackList old) { + final Map forReuse = new HashMap<>(); + for (Pack p : old.packs) { + if (p.invalid()) { + // The pack instance is corrupted, and cannot be safely used + // again. Do not include it in our reuse map. + // + p.close(); + continue; + } + + final Pack prior = forReuse.put(p.getPackFile().getName(), p); + if (prior != null) { + // This should never occur. It should be impossible for us + // to have two pack files with the same name, as all of them + // came out of the same directory. If it does, we promised to + // close any PackFiles we did not reuse, so close the second, + // readers are likely to be actively using the first. + // + forReuse.put(prior.getPackFile().getName(), prior); + p.close(); + } + } + return forReuse; + } + + private Set listPackDirectory() { + final String[] nameList = directory.list(); + if (nameList == null) { + return Collections.emptySet(); + } + final Set nameSet = new HashSet<>(nameList.length << 1); + for (String name : nameList) { + if (name.startsWith("pack-")) { //$NON-NLS-1$ + nameSet.add(name); + } + } + return nameSet; + } + + static final class PackList { + /** State just before reading the pack directory. */ + final FileSnapshot snapshot; + + /** All known packs, sorted by {@link Pack#SORT}. */ + final Pack[] packs; + + PackList(FileSnapshot monitor, Pack[] packs) { + this.snapshot = monitor; + this.packs = packs; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 31686befc..942cc9674 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -34,7 +34,7 @@ /** * Access path to locate objects by {@link org.eclipse.jgit.lib.ObjectId} in a - * {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * {@link org.eclipse.jgit.internal.storage.file.Pack}. *

    * Indexes are strictly redundant information in that we can rebuild all of the * data held in the index file from the on disk representation of the pack file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java index 612b12366..87e0b44d4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java @@ -25,7 +25,7 @@ /** * Creates a table of contents to support random access by - * {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * {@link org.eclipse.jgit.internal.storage.file.Pack}. *

    * Pack index files (the .idx suffix in a pack file pair) provides * random access to any object in the pack by associating an ObjectId to the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java index a9e058888..0bceca72e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java @@ -16,11 +16,11 @@ class PackInputStream extends InputStream { private final WindowCursor wc; - private final PackFile pack; + private final Pack pack; private long pos; - PackInputStream(PackFile pack, long pos, WindowCursor wc) + PackInputStream(Pack pack, long pos, WindowCursor wc) throws IOException { this.pack = pack; this.pos = pos; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java index 2c2f7911a..482b143e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java @@ -18,7 +18,7 @@ import org.eclipse.jgit.util.FileUtils; /** - * Keeps track of a {@link org.eclipse.jgit.internal.storage.file.PackFile}'s + * Keeps track of a {@link org.eclipse.jgit.internal.storage.file.Pack}'s * associated .keep file. */ public class PackLock { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java index 4d80a0312..ee458e27b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java @@ -25,7 +25,7 @@ *

    * * @see PackIndex - * @see PackFile + * @see Pack */ public class PackReverseIndex { /** Index we were created from, and that has our ObjectId data. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java index 9dbdbc73f..2c0ade681 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java @@ -51,9 +51,6 @@ class RefDirectoryRename extends RefRename { */ private ObjectId objId; - /** True if HEAD must be moved to the destination reference. */ - private boolean updateHEAD; - /** A reference we backup {@link #objId} into during the rename. */ private RefDirectoryUpdate tmp; @@ -69,7 +66,7 @@ protected Result doRename() throws IOException { return Result.IO_FAILURE; // not supported objId = source.getOldObjectId(); - updateHEAD = needToUpdateHEAD(); + boolean updateHEAD = needToUpdateHEAD(); tmp = refdb.newTemporaryUpdate(); try (RevWalk rw = new RevWalk(refdb.getRepository())) { // First backup the source so its never unreachable. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 3e8cb3a3f..25653b3ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -33,7 +33,7 @@ import org.eclipse.jgit.util.Monitoring; /** - * Caches slices of a {@link org.eclipse.jgit.internal.storage.file.PackFile} in + * Caches slices of a {@link org.eclipse.jgit.internal.storage.file.Pack} in * memory for faster read access. *

    * The WindowCache serves as a Java based "buffer cache", loading segments of a @@ -41,7 +41,7 @@ * only tiny slices of a file, the WindowCache tries to smooth out these tiny * reads into larger block-sized IO operations. *

    - * Whenever a cache miss occurs, {@link #load(PackFile, long)} is invoked by + * Whenever a cache miss occurs, {@link #load(Pack, long)} is invoked by * exactly one thread for the given (PackFile,position) key tuple. * This is ensured by an array of locks, with the tuple hashed to a lock * instance. @@ -80,10 +80,10 @@ *

    * This cache has an implementation rule such that: *

      - *
    • {@link #load(PackFile, long)} is invoked by at most one thread at a time + *
    • {@link #load(Pack, long)} is invoked by at most one thread at a time * for a given (PackFile,position) tuple.
      • *
    • For every load() invocation there is exactly one - * {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a + * {@link #createRef(Pack, long, ByteWindow)} invocation to wrap a * SoftReference or a StrongReference around the cached entity.
      • *
    • For every Reference created by createRef() there will be * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated @@ -91,10 +91,10 @@ *
    *

    * Therefore, it is safe to perform resource accounting increments during the - * {@link #load(PackFile, long)} or - * {@link #createRef(PackFile, long, ByteWindow)} methods, and matching + * {@link #load(Pack, long)} or + * {@link #createRef(Pack, long, ByteWindow)} methods, and matching * decrements during {@link #clear(PageRef)}. Implementors may need to override - * {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional + * {@link #createRef(Pack, long, ByteWindow)} in order to embed additional * accounting information into an implementation specific * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as * the cached entity may have already been evicted by the JRE's garbage @@ -170,7 +170,7 @@ static interface StatsRecorder { * @param delta * delta of cached bytes */ - void recordOpenBytes(PackFile pack, int delta); + void recordOpenBytes(Pack pack, int delta); /** * Returns a snapshot of this recorder's stats. Note that this may be an @@ -242,7 +242,7 @@ public void recordOpenFiles(int delta) { } @Override - public void recordOpenBytes(PackFile pack, int delta) { + public void recordOpenBytes(Pack pack, int delta) { openByteCount.add(delta); String repositoryId = repositoryId(pack); LongAdder la = openByteCountPerRepository @@ -254,9 +254,8 @@ public void recordOpenBytes(PackFile pack, int delta) { } } - private static String repositoryId(PackFile pack) { - // use repository's gitdir since packfile doesn't know its - // repository + private static String repositoryId(Pack pack) { + // use repository's gitdir since Pack doesn't know its repository return pack.getPackFile().getParentFile().getParentFile() .getParent(); } @@ -380,7 +379,7 @@ public static WindowCache getInstance() { return cache.publishMBeanIfNeeded(); } - static final ByteWindow get(PackFile pack, long offset) + static final ByteWindow get(Pack pack, long offset) throws IOException { final WindowCache c = cache; final ByteWindow r = c.getOrLoad(pack, c.toStart(offset)); @@ -395,7 +394,7 @@ static final ByteWindow get(PackFile pack, long offset) return r; } - static final void purge(PackFile pack) { + static final void purge(Pack pack) { cache.removeAll(pack); } @@ -506,7 +505,7 @@ private int hash(int packHash, long off) { return packHash + (int) (off >>> windowSizeShift); } - private ByteWindow load(PackFile pack, long offset) throws IOException { + private ByteWindow load(Pack pack, long offset) throws IOException { long startTime = System.nanoTime(); if (pack.beginWindowCache()) statsRecorder.recordOpenFiles(1); @@ -525,7 +524,7 @@ private ByteWindow load(PackFile pack, long offset) throws IOException { } } - private PageRef createRef(PackFile p, long o, ByteWindow v) { + private PageRef createRef(Pack p, long o, ByteWindow v) { final PageRef ref = useStrongRefs ? new StrongRef(p, o, v, queue) : new SoftRef(p, o, v, (SoftCleanupQueue) queue); @@ -539,7 +538,7 @@ private void clear(PageRef ref) { close(ref.getPack()); } - private void close(PackFile pack) { + private void close(Pack pack) { if (pack.endWindowCache()) { statsRecorder.recordOpenFiles(-1); } @@ -578,9 +577,9 @@ private static int lockCount(WindowCacheConfig cfg) { * @return the object reference. * @throws IOException * the object reference was not in the cache and could not be - * obtained by {@link #load(PackFile, long)}. + * obtained by {@link #load(Pack, long)}. */ - private ByteWindow getOrLoad(PackFile pack, long position) + private ByteWindow getOrLoad(Pack pack, long position) throws IOException { final int slot = slot(pack, position); final Entry e1 = table.get(slot); @@ -623,7 +622,7 @@ private ByteWindow getOrLoad(PackFile pack, long position) return v; } - private ByteWindow scan(Entry n, PackFile pack, long position) { + private ByteWindow scan(Entry n, Pack pack, long position) { for (; n != null; n = n.next) { final PageRef r = n.ref; if (r.getPack() == pack && r.getPosition() == position) { @@ -704,7 +703,7 @@ private void removeAll() { /** * Clear all entries related to a single file. *

    - * Typically this method is invoked during {@link PackFile#close()}, when we + * Typically this method is invoked during {@link Pack#close()}, when we * know the pack is never going to be useful to us again (for example, it no * longer exists on disk). A concurrent reader loading an entry from this * same pack may cause the pack to become stuck in the cache anyway. @@ -712,7 +711,7 @@ private void removeAll() { * @param pack * the file to purge all entries of. */ - private void removeAll(PackFile pack) { + private void removeAll(Pack pack) { for (int s = 0; s < tableSize; s++) { final Entry e1 = table.get(s); boolean hasDead = false; @@ -733,11 +732,11 @@ private void gc() { queue.gc(); } - private int slot(PackFile pack, long position) { + private int slot(Pack pack, long position) { return (hash(pack.hash, position) >>> 1) % tableSize; } - private Lock lock(PackFile pack, long position) { + private Lock lock(Pack pack, long position) { return locks[(hash(pack.hash, position) >>> 1) % locks.length]; } @@ -799,16 +798,20 @@ private static interface PageRef { boolean kill(); /** - * Get the packfile the referenced cache page is allocated for + * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} the + * referenced cache page is allocated for * - * @return the packfile the referenced cache page is allocated for + * @return the {@link org.eclipse.jgit.internal.storage.file.Pack} the + * referenced cache page is allocated for */ - PackFile getPack(); + Pack getPack(); /** - * Get the position of the referenced cache page in the packfile + * Get the position of the referenced cache page in the + * {@link org.eclipse.jgit.internal.storage.file.Pack} * - * @return the position of the referenced cache page in the packfile + * @return the position of the referenced cache page in the + * {@link org.eclipse.jgit.internal.storage.file.Pack} */ long getPosition(); @@ -844,7 +847,7 @@ private static interface PageRef { /** A soft reference wrapped around a cached object. */ private static class SoftRef extends SoftReference implements PageRef { - private final PackFile pack; + private final Pack pack; private final long position; @@ -852,7 +855,7 @@ private static class SoftRef extends SoftReference private long lastAccess; - protected SoftRef(final PackFile pack, final long position, + protected SoftRef(final Pack pack, final long position, final ByteWindow v, final SoftCleanupQueue queue) { super(v, queue); this.pack = pack; @@ -861,7 +864,7 @@ protected SoftRef(final PackFile pack, final long position, } @Override - public PackFile getPack() { + public Pack getPack() { return pack; } @@ -900,7 +903,7 @@ public boolean isStrongRef() { private static class StrongRef implements PageRef { private ByteWindow referent; - private final PackFile pack; + private final Pack pack; private final long position; @@ -910,7 +913,7 @@ private static class StrongRef implements PageRef { private CleanupQueue queue; - protected StrongRef(final PackFile pack, final long position, + protected StrongRef(final Pack pack, final long position, final ByteWindow v, final CleanupQueue queue) { this.pack = pack; this.position = position; @@ -920,7 +923,7 @@ protected StrongRef(final PackFile pack, final long position, } @Override - public PackFile getPack() { + public Pack getPack() { return pack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 6c975708a..e7fd7b9e7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -86,7 +86,7 @@ public ObjectReader newReader() { /** {@inheritDoc} */ @Override public BitmapIndex getBitmapIndex() throws IOException { - for (PackFile pack : db.getPacks()) { + for (Pack pack : db.getPacks()) { PackBitmapIndex index = pack.getBitmapIndex(); if (index != null) return new BitmapIndexImpl(index); @@ -98,7 +98,7 @@ public BitmapIndex getBitmapIndex() throws IOException { @Override public Collection getCachedPacksAndUpdate( BitmapBuilder needBitmap) throws IOException { - for (PackFile pack : db.getPacks()) { + for (Pack pack : db.getPacks()) { PackBitmapIndex index = pack.getBitmapIndex(); if (needBitmap.removeAllOrNone(index)) return Collections. singletonList( @@ -218,7 +218,7 @@ public void writeObjects(PackOutputStream out, List list) * this cursor does not match the provider or id and the proper * window could not be acquired through the provider's cache. */ - int copy(final PackFile pack, long position, final byte[] dstbuf, + int copy(final Pack pack, long position, final byte[] dstbuf, int dstoff, final int cnt) throws IOException { final long length = pack.length; int need = cnt; @@ -239,7 +239,7 @@ public void copyPackAsIs(PackOutputStream out, CachedPack pack) ((LocalCachedPack) pack).copyAsIs(out, this); } - void copyPackAsIs(final PackFile pack, final long length, + void copyPackAsIs(final Pack pack, final long length, final PackOutputStream out) throws IOException { long position = 12; long remaining = length - (12 + 20); @@ -275,7 +275,7 @@ void copyPackAsIs(final PackFile pack, final long length, * the inflater encountered an invalid chunk of data. Data * stream corruption is likely. */ - int inflate(final PackFile pack, long position, final byte[] dstbuf, + int inflate(final Pack pack, long position, final byte[] dstbuf, boolean headerOnly) throws IOException, DataFormatException { prepareInflater(); pin(pack, position); @@ -293,7 +293,7 @@ int inflate(final PackFile pack, long position, final byte[] dstbuf, } } - ByteArrayWindow quickCopy(PackFile p, long pos, long cnt) + ByteArrayWindow quickCopy(Pack p, long pos, long cnt) throws IOException { pin(p, pos); if (window instanceof ByteArrayWindow @@ -314,7 +314,7 @@ private void prepareInflater() { inf.reset(); } - void pin(PackFile pack, long position) + void pin(Pack pack, long position) throws IOException { final ByteWindow w = window; if (w == null || !w.contains(pack, position)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java index a78f4d24d..e210acf05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.internal.storage.reftable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.PriorityQueue; @@ -215,6 +216,23 @@ public boolean next() throws IOException { } } + @Override + public void seekPastPrefix(String prefixName) throws IOException { + List entriesToAdd = new ArrayList<>(); + entriesToAdd.addAll(queue); + if (head != null) { + entriesToAdd.add(head); + } + + head = null; + queue.clear(); + + for(RefQueueEntry entry : entriesToAdd){ + entry.rc.seekPastPrefix(prefixName); + add(entry); + } + } + private RefQueueEntry poll() { RefQueueEntry e = head; if (e != null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java index d96648eb5..5e2c35088 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java @@ -28,6 +28,19 @@ public abstract class RefCursor implements AutoCloseable { */ public abstract boolean next() throws IOException; + /** + * Seeks forward to the first ref record lexicographically beyond + * {@code prefixName} that doesn't start with {@code prefixName}. If there are + * no more results, skipping some refs won't add new results. E.g if we create a + * RefCursor that returns only results with a specific prefix, skipping that + * prefix won't give results that are not part of the original prefix. + * + * @param prefixName prefix that should be skipped. All previous refs before it + * will be skipped. + * @throws java.io.IOException references cannot be read. + */ + public abstract void seekPastPrefix(String prefixName) throws IOException; + /** * Get reference at the current position. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java index 4747be354..0c1682861 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java @@ -14,10 +14,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.lib.ObjectId; @@ -265,6 +267,54 @@ public List getRefsByPrefix(String prefix) throws IOException { return Collections.unmodifiableList(all); } + /** + * Returns refs whose names start with a given prefix excluding all refs that + * start with one of the given prefixes. + * + * @param include string that names of refs should start with; may be empty. + * @param excludes strings that names of refs can't start with; may be empty. + * @return immutable list of refs whose names start with {@code include} and + * none of the strings in {@code exclude}. + * @throws java.io.IOException the reference space cannot be accessed. + */ + public List getRefsByPrefixWithExclusions(String include, Set excludes) throws IOException { + if (excludes.isEmpty()) { + return getRefsByPrefix(include); + } + List results = new ArrayList<>(); + lock.lock(); + try { + Reftable table = reader(); + Iterator excludeIterator = + excludes.stream().sorted().collect(Collectors.toList()).iterator(); + String currentExclusion = excludeIterator.hasNext() ? excludeIterator.next() : null; + try (RefCursor rc = RefDatabase.ALL.equals(include) ? table.allRefs() : table.seekRefsWithPrefix(include)) { + while (rc.next()) { + Ref ref = table.resolve(rc.getRef()); + if (ref == null || ref.getObjectId() == null) { + continue; + } + // Skip prefixes that will never see since we are already further than those + // prefixes lexicographically. + while (excludeIterator.hasNext() && !ref.getName().startsWith(currentExclusion) + && ref.getName().compareTo(currentExclusion) > 0) { + currentExclusion = excludeIterator.next(); + } + + if (currentExclusion != null && ref.getName().startsWith(currentExclusion)) { + rc.seekPastPrefix(currentExclusion); + continue; + } + results.add(ref); + } + } + } finally { + lock.unlock(); + } + + return Collections.unmodifiableList(results); + } + /** * @return whether there is a fast SHA1 to ref map. * @throws IOException in case of I/O problems. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java index 095276f57..9e2ae9160 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java @@ -508,6 +508,21 @@ public boolean next() throws IOException { } } + @Override + public void seekPastPrefix(String prefixName) throws IOException { + initRefIndex(); + byte[] key = prefixName.getBytes(UTF_8); + ByteBuffer byteBuffer = ByteBuffer.allocate(key.length + 1); + byteBuffer.put(key); + // Add the representation of the last byte lexicographically. Based on how UTF_8 + // representation works, this byte will be bigger lexicographically than any + // UTF_8 character when translated into bytes, since 0xFF can never be a part of + // a UTF_8 string. + byteBuffer.put((byte) 0xFF); + + block = seek(REF_BLOCK_TYPE, byteBuffer.array(), refIndex, 0, refEnd); + } + @Override public Ref getRef() { return ref; @@ -681,6 +696,17 @@ public boolean next() throws IOException { } } + @Override + /** + * The implementation here would not be efficient complexity-wise since it + * expected that there are a small number of refs that match the same object id. + * In such case it's better to not even use this method (as the caller might + * expect it to be efficient). + */ + public void seekPastPrefix(String prefixName) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public Ref getRef() { return ref; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java deleted file mode 100644 index 513863608..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import java.io.IOException; - -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; - -/** Update that always rejects with {@code LOCK_FAILURE}. */ -class AlwaysFailUpdate extends RefUpdate { - private final RefTreeDatabase refdb; - - AlwaysFailUpdate(RefTreeDatabase refdb, String name) { - super(new ObjectIdRef.Unpeeled(Ref.Storage.NEW, name, null)); - this.refdb = refdb; - setCheckConflicting(false); - } - - /** {@inheritDoc} */ - @Override - protected RefDatabase getRefDatabase() { - return refdb; - } - - /** {@inheritDoc} */ - @Override - protected Repository getRepository() { - return refdb.getRepository(); - } - - /** {@inheritDoc} */ - @Override - protected boolean tryLock(boolean deref) throws IOException { - return false; - } - - /** {@inheritDoc} */ - @Override - protected void unlock() { - // No locks are held here. - } - - /** {@inheritDoc} */ - @Override - protected Result doUpdate(Result desiredResult) { - return Result.LOCK_FAILURE; - } - - /** {@inheritDoc} */ - @Override - protected Result doDelete(Result desiredResult) { - return Result.LOCK_FAILURE; - } - - /** {@inheritDoc} */ - @Override - protected Result doLink(String target) { - return Result.LOCK_FAILURE; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java deleted file mode 100644 index bb06a9e0c..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.ReceiveCommand.Result; - -/** - * Command to create, update or delete an entry inside a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - *

    - * Unlike {@link org.eclipse.jgit.transport.ReceiveCommand} (which can only - * update a reference to an {@link org.eclipse.jgit.lib.ObjectId}), a RefTree - * Command can also create, modify or delete symbolic references to a target - * reference. - *

    - * RefTree Commands may wrap a {@code ReceiveCommand} to allow callers to - * process an existing ReceiveCommand against a RefTree. - *

    - * Commands should be passed into - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree#apply(java.util.Collection)} - * for processing. - */ -public class Command { - /** - * Set unprocessed commands as failed due to transaction aborted. - *

    - * If a command is still - * {@link org.eclipse.jgit.transport.ReceiveCommand.Result#NOT_ATTEMPTED} it - * will be set to - * {@link org.eclipse.jgit.transport.ReceiveCommand.Result#REJECTED_OTHER_REASON}. - * If {@code why} is non-null its contents will be used as the message for - * the first command status. - * - * @param commands - * commands to mark as failed. - * @param why - * optional message to set on the first aborted command. - */ - public static void abort(Iterable commands, @Nullable String why) { - if (why == null || why.isEmpty()) { - why = JGitText.get().transactionAborted; - } - for (Command c : commands) { - if (c.getResult() == NOT_ATTEMPTED) { - c.setResult(REJECTED_OTHER_REASON, why); - why = JGitText.get().transactionAborted; - } - } - } - - private final Ref oldRef; - private final Ref newRef; - private final ReceiveCommand cmd; - private Result result; - - /** - * Create a command to create, update or delete a reference. - *

    - * At least one of {@code oldRef} or {@code newRef} must be supplied. - * - * @param oldRef - * expected value. Null if the ref should not exist. - * @param newRef - * desired value, must be peeled if not null and not symbolic. - * Null to delete the ref. - */ - public Command(@Nullable Ref oldRef, @Nullable Ref newRef) { - this.oldRef = oldRef; - this.newRef = newRef; - this.cmd = null; - this.result = NOT_ATTEMPTED; - - if (oldRef == null && newRef == null) { - throw new IllegalArgumentException(); - } - if (newRef != null && !newRef.isPeeled() && !newRef.isSymbolic()) { - throw new IllegalArgumentException(); - } - if (oldRef != null && newRef != null - && !oldRef.getName().equals(newRef.getName())) { - throw new IllegalArgumentException(); - } - } - - /** - * Construct a RefTree command wrapped around a ReceiveCommand. - * - * @param rw - * walk instance to peel the {@code newId}. - * @param cmd - * command received from a push client. - * @throws org.eclipse.jgit.errors.MissingObjectException - * {@code oldId} or {@code newId} is missing. - * @throws java.io.IOException - * {@code oldId} or {@code newId} cannot be peeled. - */ - public Command(RevWalk rw, ReceiveCommand cmd) - throws MissingObjectException, IOException { - this.oldRef = toRef(rw, cmd.getOldId(), cmd.getOldSymref(), - cmd.getRefName(), false); - this.newRef = toRef(rw, cmd.getNewId(), cmd.getNewSymref(), - cmd.getRefName(), true); - this.cmd = cmd; - } - - static Ref toRef(RevWalk rw, ObjectId id, @Nullable String target, - String name, boolean mustExist) - throws MissingObjectException, IOException { - if (target != null) { - return new SymbolicRef(name, - new ObjectIdRef.Unpeeled(NETWORK, target, id)); - } else if (ObjectId.zeroId().equals(id)) { - return null; - } - - try { - RevObject o = rw.parseAny(id); - if (o instanceof RevTag) { - RevObject p = rw.peel(o); - return new ObjectIdRef.PeeledTag(NETWORK, name, id, p.copy()); - } - return new ObjectIdRef.PeeledNonTag(NETWORK, name, id); - } catch (MissingObjectException e) { - if (mustExist) { - throw e; - } - return new ObjectIdRef.Unpeeled(NETWORK, name, id); - } - } - - /** - * Get name of the reference affected by this command. - * - * @return name of the reference affected by this command. - */ - public String getRefName() { - if (cmd != null) { - return cmd.getRefName(); - } else if (newRef != null) { - return newRef.getName(); - } - return oldRef.getName(); - } - - /** - * Set the result of this command. - * - * @param result - * the command result. - */ - public void setResult(Result result) { - setResult(result, null); - } - - /** - * Set the result of this command. - * - * @param result - * the command result. - * @param why - * optional message explaining the result status. - */ - public void setResult(Result result, @Nullable String why) { - if (cmd != null) { - cmd.setResult(result, why); - } else { - this.result = result; - } - } - - /** - * Get result of executing this command. - * - * @return result of executing this command. - */ - public Result getResult() { - return cmd != null ? cmd.getResult() : result; - } - - /** - * Get optional message explaining command failure. - * - * @return optional message explaining command failure. - */ - @Nullable - public String getMessage() { - return cmd != null ? cmd.getMessage() : null; - } - - /** - * Old peeled reference. - * - * @return the old reference; null if the command is creating the reference. - */ - @Nullable - public Ref getOldRef() { - return oldRef; - } - - /** - * New peeled reference. - * - * @return the new reference; null if the command is deleting the reference. - */ - @Nullable - public Ref getNewRef() { - return newRef; - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - append(s, oldRef, "CREATE"); //$NON-NLS-1$ - s.append(' '); - append(s, newRef, "DELETE"); //$NON-NLS-1$ - s.append(' ').append(getRefName()); - s.append(' ').append(getResult()); - if (getMessage() != null) { - s.append(' ').append(getMessage()); - } - return s.toString(); - } - - private static void append(StringBuilder s, Ref r, String nullName) { - if (r == null) { - s.append(nullName); - } else if (r.isSymbolic()) { - s.append(r.getTarget().getName()); - } else { - ObjectId id = r.getObjectId(); - if (id != null) { - s.append(id.name()); - } - } - } - - /** - * Check the entry is consistent with either the old or the new ref. - * - * @param entry - * current entry; null if the entry does not exist. - * @return true if entry matches {@link #getOldRef()} or - * {@link #getNewRef()}; otherwise false. - */ - boolean checkRef(@Nullable DirCacheEntry entry) { - if (entry != null && entry.getRawMode() == 0) { - entry = null; - } - return check(entry, oldRef) || check(entry, newRef); - } - - private static boolean check(@Nullable DirCacheEntry cur, - @Nullable Ref exp) { - if (cur == null) { - // Does not exist, ok if oldRef does not exist. - return exp == null; - } else if (exp == null) { - // Expected to not exist, but currently exists, fail. - return false; - } - - if (exp.isSymbolic()) { - String dst = exp.getTarget().getName(); - return cur.getRawMode() == TYPE_SYMLINK - && cur.getObjectId().equals(symref(dst)); - } - - return cur.getRawMode() == TYPE_GITLINK - && cur.getObjectId().equals(exp.getObjectId()); - } - - static ObjectId symref(String s) { - @SuppressWarnings("resource") - ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); - return fmt.idFor(OBJ_BLOB, encode(s)); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java deleted file mode 100644 index 6f12e9ccb..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.R_REFS; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.GITLINK; -import static org.eclipse.jgit.lib.FileMode.SYMLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheBuilder; -import org.eclipse.jgit.dircache.DirCacheEditor; -import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; -import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.DirCacheNameConflictException; -import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.util.RawParseUtils; - -/** - * Tree of references in the reference graph. - *

    - * The root corresponds to the {@code "refs/"} subdirectory, for example the - * default reference {@code "refs/heads/master"} is stored at path - * {@code "heads/master"} in a {@code RefTree}. - *

    - * Normal references are stored as {@link org.eclipse.jgit.lib.FileMode#GITLINK} - * tree entries. The ObjectId in the tree entry is the ObjectId the reference - * refers to. - *

    - * Symbolic references are stored as - * {@link org.eclipse.jgit.lib.FileMode#SYMLINK} entries, with the blob storing - * the name of the target reference. - *

    - * Annotated tags also store the peeled object using a {@code GITLINK} entry - * with the suffix " ^" (space carrot), for example - * {@code "tags/v1.0"} stores the annotated tag object, while - * "tags/v1.0 ^" stores the commit the tag annotates. - *

    - * {@code HEAD} is a special case and stored as {@code "..HEAD"}. - */ -public class RefTree { - /** Suffix applied to GITLINK to indicate its the peeled value of a tag. */ - public static final String PEELED_SUFFIX = " ^"; //$NON-NLS-1$ - static final String ROOT_DOTDOT = ".."; //$NON-NLS-1$ - - /** - * Create an empty reference tree. - * - * @return a new empty reference tree. - */ - public static RefTree newEmptyTree() { - return new RefTree(DirCache.newInCore()); - } - - /** - * Load a reference tree. - * - * @param reader - * reader to scan the reference tree with. - * @param tree - * the tree to read. - * @return the ref tree read from the commit. - * @throws java.io.IOException - * the repository cannot be accessed through the reader. - * @throws org.eclipse.jgit.errors.CorruptObjectException - * a tree object is corrupt and cannot be read. - * @throws org.eclipse.jgit.errors.IncorrectObjectTypeException - * a tree object wasn't actually a tree. - * @throws org.eclipse.jgit.errors.MissingObjectException - * a reference tree object doesn't exist. - */ - public static RefTree read(ObjectReader reader, RevTree tree) - throws MissingObjectException, IncorrectObjectTypeException, - CorruptObjectException, IOException { - return new RefTree(DirCache.read(reader, tree)); - } - - private DirCache contents; - private Map pendingBlobs; - - private RefTree(DirCache dc) { - this.contents = dc; - } - - /** - * Read one reference. - *

    - * References are always returned peeled - * ({@link org.eclipse.jgit.lib.Ref#isPeeled()} is true). If the reference - * points to an annotated tag, the returned reference will be peeled and - * contain {@link org.eclipse.jgit.lib.Ref#getPeeledObjectId()}. - *

    - * If the reference is a symbolic reference and the chain depth is less than - * {@link org.eclipse.jgit.lib.RefDatabase#MAX_SYMBOLIC_REF_DEPTH} the - * returned reference is resolved. If the chain depth is longer, the - * symbolic reference is returned without resolving. - * - * @param reader - * to access objects necessary to read the requested reference. - * @param name - * name of the reference to read. - * @return the reference; null if it does not exist. - * @throws java.io.IOException - * cannot read a symbolic reference target. - */ - @Nullable - public Ref exactRef(ObjectReader reader, String name) throws IOException { - Ref r = readRef(reader, name); - if (r == null) { - return null; - } else if (r.isSymbolic()) { - return resolve(reader, r, 0); - } - - DirCacheEntry p = contents.getEntry(peeledPath(name)); - if (p != null && p.getRawMode() == TYPE_GITLINK) { - return new ObjectIdRef.PeeledTag(PACKED, r.getName(), - r.getObjectId(), p.getObjectId()); - } - return r; - } - - private Ref readRef(ObjectReader reader, String name) throws IOException { - DirCacheEntry e = contents.getEntry(refPath(name)); - return e != null ? toRef(reader, e, name) : null; - } - - private Ref toRef(ObjectReader reader, DirCacheEntry e, String name) - throws IOException { - int mode = e.getRawMode(); - if (mode == TYPE_GITLINK) { - ObjectId id = e.getObjectId(); - return new ObjectIdRef.PeeledNonTag(PACKED, name, id); - } - - if (mode == TYPE_SYMLINK) { - ObjectId id = e.getObjectId(); - String n = pendingBlobs != null ? pendingBlobs.get(id) : null; - if (n == null) { - byte[] bin = reader.open(id, OBJ_BLOB).getCachedBytes(); - n = RawParseUtils.decode(bin); - } - Ref dst = new ObjectIdRef.Unpeeled(NEW, n, null); - return new SymbolicRef(name, dst); - } - - return null; // garbage file or something; not a reference. - } - - private Ref resolve(ObjectReader reader, Ref ref, int depth) - throws IOException { - if (ref.isSymbolic() && depth < MAX_SYMBOLIC_REF_DEPTH) { - Ref r = readRef(reader, ref.getTarget().getName()); - if (r == null) { - return ref; - } - Ref dst = resolve(reader, r, depth + 1); - return new SymbolicRef(ref.getName(), dst); - } - return ref; - } - - /** - * Attempt a batch of commands against this RefTree. - *

    - * The batch is applied atomically, either all commands apply at once, or - * they all reject and the RefTree is left unmodified. - *

    - * On success (when this method returns {@code true}) the command results - * are left as-is (probably {@code NOT_ATTEMPTED}). Result fields are set - * only when this method returns {@code false} to indicate failure. - * - * @param cmdList - * to apply. All commands should still have result NOT_ATTEMPTED. - * @return true if the commands applied; false if they were rejected. - */ - public boolean apply(Collection cmdList) { - try { - DirCacheEditor ed = contents.editor(); - for (Command cmd : cmdList) { - if (!isValidRef(cmd)) { - cmd.setResult(REJECTED_OTHER_REASON, - JGitText.get().funnyRefname); - Command.abort(cmdList, null); - return false; - } - apply(ed, cmd); - } - ed.finish(); - return true; - } catch (DirCacheNameConflictException e) { - String r1 = refName(e.getPath1()); - String r2 = refName(e.getPath2()); - for (Command cmd : cmdList) { - if (r1.equals(cmd.getRefName()) - || r2.equals(cmd.getRefName())) { - cmd.setResult(LOCK_FAILURE); - break; - } - } - Command.abort(cmdList, null); - return false; - } catch (LockFailureException e) { - Command.abort(cmdList, null); - return false; - } - } - - private static boolean isValidRef(Command cmd) { - String n = cmd.getRefName(); - return HEAD.equals(n) || Repository.isValidRefName(n); - } - - private void apply(DirCacheEditor ed, Command cmd) { - String path = refPath(cmd.getRefName()); - Ref oldRef = cmd.getOldRef(); - final Ref newRef = cmd.getNewRef(); - - if (newRef == null) { - checkRef(contents.getEntry(path), cmd); - ed.add(new DeletePath(path)); - cleanupPeeledRef(ed, oldRef); - return; - } - - if (newRef.isSymbolic()) { - final String dst = newRef.getTarget().getName(); - ed.add(new PathEdit(path) { - @Override - public void apply(DirCacheEntry ent) { - checkRef(ent, cmd); - ObjectId id = Command.symref(dst); - ent.setFileMode(SYMLINK); - ent.setObjectId(id); - if (pendingBlobs == null) { - pendingBlobs = new HashMap<>(4); - } - pendingBlobs.put(id, dst); - } - }.setReplace(false)); - cleanupPeeledRef(ed, oldRef); - return; - } - - ed.add(new PathEdit(path) { - @Override - public void apply(DirCacheEntry ent) { - checkRef(ent, cmd); - ent.setFileMode(GITLINK); - ent.setObjectId(newRef.getObjectId()); - } - }.setReplace(false)); - - if (newRef.getPeeledObjectId() != null) { - ed.add(new PathEdit(peeledPath(newRef.getName())) { - @Override - public void apply(DirCacheEntry ent) { - ent.setFileMode(GITLINK); - ent.setObjectId(newRef.getPeeledObjectId()); - } - }.setReplace(false)); - } else { - cleanupPeeledRef(ed, oldRef); - } - } - - private static void checkRef(@Nullable DirCacheEntry ent, Command cmd) { - if (!cmd.checkRef(ent)) { - cmd.setResult(LOCK_FAILURE); - throw new LockFailureException(); - } - } - - private static void cleanupPeeledRef(DirCacheEditor ed, Ref ref) { - if (ref != null && !ref.isSymbolic() - && (!ref.isPeeled() || ref.getPeeledObjectId() != null)) { - ed.add(new DeletePath(peeledPath(ref.getName()))); - } - } - - /** - * Convert a path name in a RefTree to the reference name known by Git. - * - * @param path - * name read from the RefTree structure, for example - * {@code "heads/master"}. - * @return reference name for the path, {@code "refs/heads/master"}. - */ - public static String refName(String path) { - if (path.startsWith(ROOT_DOTDOT)) { - return path.substring(2); - } - return R_REFS + path; - } - - static String refPath(String name) { - if (name.startsWith(R_REFS)) { - return name.substring(R_REFS.length()); - } - return ROOT_DOTDOT + name; - } - - private static String peeledPath(String name) { - return refPath(name) + PEELED_SUFFIX; - } - - /** - * Write this reference tree. - * - * @param inserter - * inserter to use when writing trees to the object database. - * Caller is responsible for flushing the inserter before trying - * to read the objects, or exposing them through a reference. - * @return the top level tree. - * @throws java.io.IOException - * a tree could not be written. - */ - public ObjectId writeTree(ObjectInserter inserter) throws IOException { - if (pendingBlobs != null) { - for (String s : pendingBlobs.values()) { - inserter.insert(OBJ_BLOB, encode(s)); - } - pendingBlobs = null; - } - return contents.writeTree(inserter); - } - - /** - * Create a deep copy of this RefTree. - * - * @return a deep copy of this RefTree. - */ - public RefTree copy() { - RefTree r = new RefTree(DirCache.newInCore()); - DirCacheBuilder b = r.contents.builder(); - for (int i = 0; i < contents.getEntryCount(); i++) { - b.add(new DirCacheEntry(contents.getEntry(i))); - } - b.finish(); - if (pendingBlobs != null) { - r.pendingBlobs = new HashMap<>(pendingBlobs); - } - return r; - } - - private static class LockFailureException extends RuntimeException { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java deleted file mode 100644 index b154b95ad..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_TREE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** Batch update a {@link RefTreeDatabase}. */ -class RefTreeBatch extends BatchRefUpdate { - private final RefTreeDatabase refdb; - private Ref src; - private ObjectId parentCommitId; - private ObjectId parentTreeId; - private RefTree tree; - private PersonIdent author; - private ObjectId newCommitId; - - RefTreeBatch(RefTreeDatabase refdb) { - super(refdb); - this.refdb = refdb; - } - - /** {@inheritDoc} */ - @Override - public void execute(RevWalk rw, ProgressMonitor monitor) - throws IOException { - List todo = new ArrayList<>(getCommands().size()); - for (ReceiveCommand c : getCommands()) { - if (!isAllowNonFastForwards()) { - if (c.getType() == UPDATE) { - c.updateType(rw); - } - if (c.getType() == UPDATE_NONFASTFORWARD) { - c.setResult(REJECTED_NONFASTFORWARD); - if (isAtomic()) { - ReceiveCommand.abort(getCommands()); - return; - } - continue; - } - } - todo.add(new Command(rw, c)); - } - init(rw); - execute(rw, todo); - } - - void init(RevWalk rw) throws IOException { - src = refdb.getBootstrap().exactRef(refdb.getTxnCommitted()); - if (src != null && src.getObjectId() != null) { - RevCommit c = rw.parseCommit(src.getObjectId()); - parentCommitId = c; - parentTreeId = c.getTree(); - tree = RefTree.read(rw.getObjectReader(), c.getTree()); - } else { - parentCommitId = ObjectId.zeroId(); - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - parentTreeId = fmt.idFor(OBJ_TREE, new byte[] {}); - } - tree = RefTree.newEmptyTree(); - } - } - - @Nullable - Ref exactRef(ObjectReader reader, String name) throws IOException { - return tree.exactRef(reader, name); - } - - /** - * Execute an update from {@link RefTreeUpdate} or {@link RefTreeRename}. - * - * @param rw - * current RevWalk handling the update or rename. - * @param todo - * commands to execute. Must never be a bootstrap reference name. - * @throws IOException - * the storage system is unable to read or write data. - */ - void execute(RevWalk rw, List todo) throws IOException { - for (Command c : todo) { - if (c.getResult() != NOT_ATTEMPTED) { - Command.abort(todo, null); - return; - } - if (refdb.conflictsWithBootstrap(c.getRefName())) { - c.setResult(REJECTED_OTHER_REASON, MessageFormat - .format(JGitText.get().invalidRefName, c.getRefName())); - Command.abort(todo, null); - return; - } - } - - if (apply(todo) && newCommitId != null) { - commit(rw, todo); - } - } - - private boolean apply(List todo) throws IOException { - if (!tree.apply(todo)) { - // apply set rejection information on commands. - return false; - } - - Repository repo = refdb.getRepository(); - try (ObjectInserter ins = repo.newObjectInserter()) { - CommitBuilder b = new CommitBuilder(); - b.setTreeId(tree.writeTree(ins)); - if (parentTreeId.equals(b.getTreeId())) { - for (Command c : todo) { - c.setResult(OK); - } - return true; - } - if (!parentCommitId.equals(ObjectId.zeroId())) { - b.setParentId(parentCommitId); - } - - author = getRefLogIdent(); - if (author == null) { - author = new PersonIdent(repo); - } - b.setAuthor(author); - b.setCommitter(author); - b.setMessage(getRefLogMessage()); - newCommitId = ins.insert(b); - ins.flush(); - } - return true; - } - - private void commit(RevWalk rw, List todo) throws IOException { - ReceiveCommand commit = new ReceiveCommand( - parentCommitId, newCommitId, - refdb.getTxnCommitted()); - updateBootstrap(rw, commit); - - if (commit.getResult() == OK) { - for (Command c : todo) { - c.setResult(OK); - } - } else { - Command.abort(todo, commit.getResult().name()); - } - } - - private void updateBootstrap(RevWalk rw, ReceiveCommand commit) - throws IOException { - BatchRefUpdate u = refdb.getBootstrap().newBatchUpdate(); - u.setAllowNonFastForwards(true); - u.setPushCertificate(getPushCertificate()); - if (isRefLogDisabled()) { - u.disableRefLog(); - } else { - u.setRefLogIdent(author); - u.setRefLogMessage(getRefLogMessage(), false); - } - u.addCommand(commit); - u.execute(rw, NullProgressMonitor.INSTANCE); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java deleted file mode 100644 index 34b8f2cd3..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Ref.Storage; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefRename; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.RefList; -import org.eclipse.jgit.util.RefMap; - -/** - * Reference database backed by a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - *

    - * The storage for RefTreeDatabase has two parts. The main part is a native Git - * tree object stored under the {@code refs/txn} namespace. To avoid cycles, - * references to {@code refs/txn} are not stored in that tree object, but - * instead in a "bootstrap" layer, which is a separate - * {@link org.eclipse.jgit.lib.RefDatabase} such as - * {@link org.eclipse.jgit.internal.storage.file.RefDirectory} using local - * reference files inside of {@code $GIT_DIR/refs}. - */ -public class RefTreeDatabase extends RefDatabase { - private final Repository repo; - private final RefDatabase bootstrap; - private final String txnCommitted; - - @Nullable - private final String txnNamespace; - private volatile Scanner.Result refs; - - /** - * Create a RefTreeDb for a repository. - * - * @param repo - * the repository using references in this database. - * @param bootstrap - * bootstrap reference database storing the references that - * anchor the - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - */ - public RefTreeDatabase(Repository repo, RefDatabase bootstrap) { - Config cfg = repo.getConfig(); - String committed = cfg.getString("reftree", null, "committedRef"); //$NON-NLS-1$ //$NON-NLS-2$ - if (committed == null || committed.isEmpty()) { - committed = "refs/txn/committed"; //$NON-NLS-1$ - } - - this.repo = repo; - this.bootstrap = bootstrap; - this.txnNamespace = initNamespace(committed); - this.txnCommitted = committed; - } - - /** - * Create a RefTreeDb for a repository. - * - * @param repo - * the repository using references in this database. - * @param bootstrap - * bootstrap reference database storing the references that - * anchor the - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - * @param txnCommitted - * name of the bootstrap reference holding the committed RefTree. - */ - public RefTreeDatabase(Repository repo, RefDatabase bootstrap, - String txnCommitted) { - this.repo = repo; - this.bootstrap = bootstrap; - this.txnNamespace = initNamespace(txnCommitted); - this.txnCommitted = txnCommitted; - } - - private static String initNamespace(String committed) { - int s = committed.lastIndexOf('/'); - if (s < 0) { - return null; - } - return committed.substring(0, s + 1); // Keep trailing '/'. - } - - Repository getRepository() { - return repo; - } - - /** - * Get the bootstrap reference database - * - * @return the bootstrap reference database, which must be used to access - * {@link #getTxnCommitted()}, {@link #getTxnNamespace()}. - */ - public RefDatabase getBootstrap() { - return bootstrap; - } - - /** - * Get name of bootstrap reference anchoring committed RefTree. - * - * @return name of bootstrap reference anchoring committed RefTree. - */ - public String getTxnCommitted() { - return txnCommitted; - } - - /** - * Get namespace used by bootstrap layer. - * - * @return namespace used by bootstrap layer, e.g. {@code refs/txn/}. Always - * ends in {@code '/'}. - */ - @Nullable - public String getTxnNamespace() { - return txnNamespace; - } - - /** {@inheritDoc} */ - @Override - public void create() throws IOException { - bootstrap.create(); - } - - /** {@inheritDoc} */ - @Override - public boolean performsAtomicTransactions() { - return true; - } - - /** {@inheritDoc} */ - @Override - public void refresh() { - bootstrap.refresh(); - } - - /** {@inheritDoc} */ - @Override - public void close() { - refs = null; - bootstrap.close(); - } - - /** {@inheritDoc} */ - @Override - public Ref exactRef(String name) throws IOException { - if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { - // Pass through names like MERGE_HEAD, ORIG_HEAD, FETCH_HEAD. - return bootstrap.exactRef(name); - } else if (conflictsWithBootstrap(name)) { - return null; - } - - boolean partial = false; - Ref src = bootstrap.exactRef(txnCommitted); - Scanner.Result c = refs; - if (c == null || !c.refTreeId.equals(idOf(src))) { - c = Scanner.scanRefTree(repo, src, prefixOf(name), false); - partial = true; - } - - Ref r = c.all.get(name); - if (r != null && r.isSymbolic()) { - r = c.sym.get(name); - if (partial && r.getObjectId() == null) { - // Attempting exactRef("HEAD") with partial scan will leave - // an unresolved symref as its target e.g. refs/heads/master - // was not read by the partial scan. Scan everything instead. - return getRefs(ALL).get(name); - } - } - return r; - } - - private static String prefixOf(String name) { - int s = name.lastIndexOf('/'); - if (s >= 0) { - return name.substring(0, s); - } - return ""; //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @Override - public Map getRefs(String prefix) throws IOException { - if (!prefix.isEmpty() && prefix.charAt(prefix.length() - 1) != '/') { - return new HashMap<>(0); - } - - Ref src = bootstrap.exactRef(txnCommitted); - Scanner.Result c = refs; - if (c == null || !c.refTreeId.equals(idOf(src))) { - c = Scanner.scanRefTree(repo, src, prefix, true); - if (prefix.isEmpty()) { - refs = c; - } - } - return new RefMap(prefix, RefList. emptyList(), c.all, c.sym); - } - - private static ObjectId idOf(@Nullable Ref src) { - return src != null && src.getObjectId() != null - ? src.getObjectId() - : ObjectId.zeroId(); - } - - /** {@inheritDoc} */ - @Override - public List getAdditionalRefs() throws IOException { - Collection txnRefs; - if (txnNamespace != null) { - txnRefs = bootstrap.getRefsByPrefix(txnNamespace); - } else { - Ref r = bootstrap.exactRef(txnCommitted); - if (r != null && r.getObjectId() != null) { - txnRefs = Collections.singleton(r); - } else { - txnRefs = Collections.emptyList(); - } - } - - List otherRefs = bootstrap.getAdditionalRefs(); - List all = new ArrayList<>(txnRefs.size() + otherRefs.size()); - all.addAll(txnRefs); - all.addAll(otherRefs); - return all; - } - - /** {@inheritDoc} */ - @Override - public Ref peel(Ref ref) throws IOException { - Ref i = ref.getLeaf(); - ObjectId id = i.getObjectId(); - if (i.isPeeled() || id == null) { - return ref; - } - try (RevWalk rw = new RevWalk(repo)) { - RevObject obj = rw.parseAny(id); - if (obj instanceof RevTag) { - ObjectId p = rw.peel(obj).copy(); - i = new ObjectIdRef.PeeledTag(PACKED, i.getName(), id, p); - } else { - i = new ObjectIdRef.PeeledNonTag(PACKED, i.getName(), id); - } - } - return recreate(ref, i); - } - - private static Ref recreate(Ref old, Ref leaf) { - if (old.isSymbolic()) { - Ref dst = recreate(old.getTarget(), leaf); - return new SymbolicRef(old.getName(), dst); - } - return leaf; - } - - /** {@inheritDoc} */ - @Override - public boolean isNameConflicting(String name) throws IOException { - return conflictsWithBootstrap(name) - || !getConflictingNames(name).isEmpty(); - } - - /** {@inheritDoc} */ - @Override - public BatchRefUpdate newBatchUpdate() { - return new RefTreeBatch(this); - } - - /** {@inheritDoc} */ - @Override - public RefUpdate newUpdate(String name, boolean detach) throws IOException { - if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { - return bootstrap.newUpdate(name, detach); - } - if (conflictsWithBootstrap(name)) { - return new AlwaysFailUpdate(this, name); - } - - Ref r = exactRef(name); - if (r == null) { - r = new ObjectIdRef.Unpeeled(Storage.NEW, name, null); - } - - boolean detaching = detach && r.isSymbolic(); - if (detaching) { - r = new ObjectIdRef.Unpeeled(LOOSE, name, r.getObjectId()); - } - - RefTreeUpdate u = new RefTreeUpdate(this, r); - if (detaching) { - u.setDetachingSymbolicRef(); - } - return u; - } - - /** {@inheritDoc} */ - @Override - public RefRename newRename(String fromName, String toName) - throws IOException { - RefUpdate from = newUpdate(fromName, true); - RefUpdate to = newUpdate(toName, true); - return new RefTreeRename(this, from, to); - } - - boolean conflictsWithBootstrap(String name) { - if (txnNamespace != null && name.startsWith(txnNamespace)) { - return true; - } else if (txnCommitted.equals(name)) { - return true; - } - - if (name.indexOf('/') < 0 && !HEAD.equals(name)) { - return true; - } - - if (name.length() > txnCommitted.length() - && name.charAt(txnCommitted.length()) == '/' - && name.startsWith(txnCommitted)) { - return true; - } - return false; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java deleted file mode 100644 index eec0da29e..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import org.eclipse.jgit.lib.RefDatabase; - -/** - * Magic reference name logic for RefTrees. - */ -public class RefTreeNames { - /** - * Suffix used on a {@link RefTreeDatabase#getTxnNamespace()} for user data. - *

    - * A {@link RefTreeDatabase}'s namespace may include a subspace (e.g. - * {@code "refs/txn/stage/"}) containing commit objects from the usual user - * portion of the repository (e.g. {@code "refs/heads/"}). These should be - * packed by the garbage collector alongside other user content rather than - * with the RefTree. - */ - private static final String STAGE = "stage/"; //$NON-NLS-1$ - - /** - * Determine if the reference is likely to be a RefTree. - * - * @param refdb - * database instance. - * @param ref - * reference name. - * @return {@code true} if the reference is a RefTree. - */ - public static boolean isRefTree(RefDatabase refdb, String ref) { - if (refdb instanceof RefTreeDatabase) { - RefTreeDatabase b = (RefTreeDatabase) refdb; - if (ref.equals(b.getTxnCommitted())) { - return true; - } - - String namespace = b.getTxnNamespace(); - if (namespace != null - && ref.startsWith(namespace) - && !ref.startsWith(namespace + STAGE)) { - return true; - } - } - return false; - } - - private RefTreeNames() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java deleted file mode 100644 index ccf8f75e7..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.RefUpdate.Result.REJECTED; -import static org.eclipse.jgit.lib.RefUpdate.Result.RENAMED; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefRename; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.RefUpdate.Result; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevWalk; - -/** Single reference rename to {@link RefTreeDatabase}. */ -class RefTreeRename extends RefRename { - private final RefTreeDatabase refdb; - - RefTreeRename(RefTreeDatabase refdb, RefUpdate src, RefUpdate dst) { - super(src, dst); - this.refdb = refdb; - } - - /** {@inheritDoc} */ - @Override - protected Result doRename() throws IOException { - try (RevWalk rw = new RevWalk(refdb.getRepository())) { - RefTreeBatch batch = new RefTreeBatch(refdb); - batch.setRefLogIdent(getRefLogIdent()); - batch.setRefLogMessage(getRefLogMessage(), false); - batch.init(rw); - - Ref head = batch.exactRef(rw.getObjectReader(), HEAD); - Ref oldRef = batch.exactRef(rw.getObjectReader(), source.getName()); - if (oldRef == null) { - return REJECTED; - } - - Ref newRef = asNew(oldRef); - List mv = new ArrayList<>(3); - mv.add(new Command(oldRef, null)); - mv.add(new Command(null, newRef)); - if (head != null && head.isSymbolic() - && head.getTarget().getName().equals(oldRef.getName())) { - mv.add(new Command( - head, - new SymbolicRef(head.getName(), newRef))); - } - batch.execute(rw, mv); - return RefTreeUpdate.translate(mv.get(1).getResult(), RENAMED); - } - } - - private Ref asNew(Ref src) { - String name = destination.getName(); - if (src.isSymbolic()) { - return new SymbolicRef(name, src.getTarget()); - } - - ObjectId peeled = src.getPeeledObjectId(); - if (peeled != null) { - return new ObjectIdRef.PeeledTag( - src.getStorage(), - name, - src.getObjectId(), - peeled); - } - - return new ObjectIdRef.PeeledNonTag( - src.getStorage(), - name, - src.getObjectId()); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java deleted file mode 100644 index 6e6ccd998..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; - -import java.io.IOException; -import java.util.Collections; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** Single reference update to {@link RefTreeDatabase}. */ -class RefTreeUpdate extends RefUpdate { - private final RefTreeDatabase refdb; - private RevWalk rw; - private RefTreeBatch batch; - private Ref oldRef; - - RefTreeUpdate(RefTreeDatabase refdb, Ref ref) { - super(ref); - this.refdb = refdb; - setCheckConflicting(false); // Done automatically by doUpdate. - } - - /** {@inheritDoc} */ - @Override - protected RefDatabase getRefDatabase() { - return refdb; - } - - /** {@inheritDoc} */ - @Override - protected Repository getRepository() { - return refdb.getRepository(); - } - - /** {@inheritDoc} */ - @Override - protected boolean tryLock(boolean deref) throws IOException { - rw = new RevWalk(getRepository()); - batch = new RefTreeBatch(refdb); - batch.init(rw); - oldRef = batch.exactRef(rw.getObjectReader(), getName()); - if (oldRef != null && oldRef.getObjectId() != null) { - setOldObjectId(oldRef.getObjectId()); - } else if (oldRef == null && getExpectedOldObjectId() != null) { - setOldObjectId(ObjectId.zeroId()); - } - return true; - } - - /** {@inheritDoc} */ - @Override - protected void unlock() { - batch = null; - if (rw != null) { - rw.close(); - rw = null; - } - } - - /** {@inheritDoc} */ - @Override - protected Result doUpdate(Result desiredResult) throws IOException { - return run(newRef(getName(), getNewObjectId()), desiredResult); - } - - private Ref newRef(String name, ObjectId id) - throws MissingObjectException, IOException { - RevObject o = rw.parseAny(id); - if (o instanceof RevTag) { - RevObject p = rw.peel(o); - return new ObjectIdRef.PeeledTag(LOOSE, name, id, p.copy()); - } - return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); - } - - /** {@inheritDoc} */ - @Override - protected Result doDelete(Result desiredResult) throws IOException { - return run(null, desiredResult); - } - - /** {@inheritDoc} */ - @Override - protected Result doLink(String target) throws IOException { - Ref dst = new ObjectIdRef.Unpeeled(NEW, target, null); - SymbolicRef n = new SymbolicRef(getName(), dst); - Result desiredResult = getRef().getStorage() == NEW - ? Result.NEW - : Result.FORCED; - return run(n, desiredResult); - } - - private Result run(@Nullable Ref newRef, Result desiredResult) - throws IOException { - Command c = new Command(oldRef, newRef); - batch.setRefLogIdent(getRefLogIdent()); - batch.setRefLogMessage(getRefLogMessage(), isRefLogIncludingResult()); - batch.execute(rw, Collections.singletonList(c)); - return translate(c.getResult(), desiredResult); - } - - static Result translate(ReceiveCommand.Result r, Result desiredResult) { - switch (r) { - case OK: - return desiredResult; - - case LOCK_FAILURE: - return Result.LOCK_FAILURE; - - case NOT_ATTEMPTED: - return Result.NOT_ATTEMPTED; - - case REJECTED_MISSING_OBJECT: - return Result.IO_FAILURE; - - case REJECTED_CURRENT_BRANCH: - return Result.REJECTED_CURRENT_BRANCH; - - case REJECTED_OTHER_REASON: - case REJECTED_NOCREATE: - case REJECTED_NODELETE: - case REJECTED_NONFASTFORWARD: - default: - return Result.REJECTED; - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java deleted file mode 100644 index 3f5122911..000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.R_REFS; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; - -import java.io.IOException; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.treewalk.AbstractTreeIterator; -import org.eclipse.jgit.treewalk.CanonicalTreeParser; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.Paths; -import org.eclipse.jgit.util.RawParseUtils; -import org.eclipse.jgit.util.RefList; - -/** A tree parser that extracts references from a {@link RefTree}. */ -class Scanner { - private static final int MAX_SYMLINK_BYTES = 10 << 10; - private static final byte[] BINARY_R_REFS = encode(R_REFS); - private static final byte[] REFS_DOT_DOT = encode("refs/.."); //$NON-NLS-1$ - - static class Result { - final ObjectId refTreeId; - final RefList all; - final RefList sym; - - Result(ObjectId id, RefList all, RefList sym) { - this.refTreeId = id; - this.all = all; - this.sym = sym; - } - } - - /** - * Scan a {@link RefTree} and parse entries into {@link Ref} instances. - * - * @param repo - * source repository containing the commit and tree objects that - * make up the RefTree. - * @param src - * bootstrap reference such as {@code refs/txn/committed} to read - * the reference tree tip from. The current ObjectId will be - * included in {@link Result#refTreeId}. - * @param prefix - * if non-empty a reference prefix to scan only a subdirectory. - * For example {@code prefix = "refs/heads/"} will limit the scan - * to only the {@code "heads"} directory of the RefTree, avoiding - * other directories like {@code "tags"}. Empty string reads all - * entries in the RefTree. - * @param recursive - * if true recurse into subdirectories of the reference tree; - * false to read only one level. Callers may use false during an - * implementation of {@code exactRef(String)} where only one - * reference is needed out of a specific subtree. - * @return sorted list of references after parsing. - * @throws IOException - * tree cannot be accessed from the repository. - */ - static Result scanRefTree(Repository repo, @Nullable Ref src, String prefix, - boolean recursive) throws IOException { - RefList.Builder all = new RefList.Builder<>(); - RefList.Builder sym = new RefList.Builder<>(); - - ObjectId srcId; - if (src != null && src.getObjectId() != null) { - try (ObjectReader reader = repo.newObjectReader()) { - srcId = src.getObjectId(); - scan(reader, srcId, prefix, recursive, all, sym); - } - } else { - srcId = ObjectId.zeroId(); - } - - RefList aList = all.toRefList(); - for (int idx = 0; idx < sym.size();) { - Ref s = sym.get(idx); - Ref r = resolve(s, 0, aList); - if (r != null) { - sym.set(idx++, r); - } else { - // Remove broken symbolic reference, they don't exist. - sym.remove(idx); - int rm = aList.find(s.getName()); - if (0 <= rm) { - aList = aList.remove(rm); - } - } - } - return new Result(srcId, aList, sym.toRefList()); - } - - private static void scan(ObjectReader reader, AnyObjectId srcId, - String prefix, boolean recursive, - RefList.Builder all, RefList.Builder sym) - throws IncorrectObjectTypeException, IOException { - CanonicalTreeParser p = createParserAtPath(reader, srcId, prefix); - if (p == null) { - return; - } - - while (!p.eof()) { - int mode = p.getEntryRawMode(); - if (mode == TYPE_TREE) { - if (recursive) { - p = p.createSubtreeIterator(reader); - } else { - p = p.next(); - } - continue; - } - - if (!curElementHasPeelSuffix(p)) { - Ref r = toRef(reader, mode, p); - if (r != null) { - all.add(r); - if (r.isSymbolic()) { - sym.add(r); - } - } - } else if (mode == TYPE_GITLINK) { - peel(all, p); - } - p = p.next(); - } - } - - private static CanonicalTreeParser createParserAtPath(ObjectReader reader, - AnyObjectId srcId, String prefix) throws IOException { - ObjectId root = toTree(reader, srcId); - if (prefix.isEmpty()) { - return new CanonicalTreeParser(BINARY_R_REFS, reader, root); - } - - String dir = RefTree.refPath(Paths.stripTrailingSeparator(prefix)); - TreeWalk tw = TreeWalk.forPath(reader, dir, root); - if (tw == null || !tw.isSubtree()) { - return null; - } - - ObjectId id = tw.getObjectId(0); - return new CanonicalTreeParser(encode(prefix), reader, id); - } - - private static Ref resolve(Ref ref, int depth, RefList refs) - throws IOException { - if (!ref.isSymbolic()) { - return ref; - } else if (MAX_SYMBOLIC_REF_DEPTH <= depth) { - return null; - } - - Ref r = refs.get(ref.getTarget().getName()); - if (r == null) { - return ref; - } - - Ref dst = resolve(r, depth + 1, refs); - if (dst == null) { - return null; - } - return new SymbolicRef(ref.getName(), dst); - } - - private static RevTree toTree(ObjectReader reader, AnyObjectId id) - throws IOException { - try (RevWalk rw = new RevWalk(reader)) { - return rw.parseTree(id); - } - } - - private static boolean curElementHasPeelSuffix(AbstractTreeIterator itr) { - int n = itr.getEntryPathLength(); - byte[] c = itr.getEntryPathBuffer(); - return n > 2 && c[n - 2] == ' ' && c[n - 1] == '^'; - } - - private static void peel(RefList.Builder all, CanonicalTreeParser p) { - String name = refName(p, true); - for (int idx = all.size() - 1; 0 <= idx; idx--) { - Ref r = all.get(idx); - int cmp = r.getName().compareTo(name); - if (cmp == 0) { - all.set(idx, new ObjectIdRef.PeeledTag(PACKED, r.getName(), - r.getObjectId(), p.getEntryObjectId())); - break; - } else if (cmp < 0) { - // Stray peeled name without matching base name; skip entry. - break; - } - } - } - - private static Ref toRef(ObjectReader reader, int mode, - CanonicalTreeParser p) throws IOException { - if (mode == TYPE_GITLINK) { - String name = refName(p, false); - ObjectId id = p.getEntryObjectId(); - return new ObjectIdRef.PeeledNonTag(PACKED, name, id); - - } else if (mode == TYPE_SYMLINK) { - ObjectId id = p.getEntryObjectId(); - byte[] bin = reader.open(id, OBJ_BLOB) - .getCachedBytes(MAX_SYMLINK_BYTES); - String dst = RawParseUtils.decode(bin); - Ref trg = new ObjectIdRef.Unpeeled(NEW, dst, null); - String name = refName(p, false); - return new SymbolicRef(name, trg); - } - return null; - } - - private static String refName(CanonicalTreeParser p, boolean peel) { - byte[] buf = p.getEntryPathBuffer(); - int len = p.getEntryPathLength(); - if (peel) { - len -= 2; - } - int ptr = 0; - if (RawParseUtils.match(buf, ptr, REFS_DOT_DOT) > 0) { - ptr = 7; - } - return RawParseUtils.decode(buf, ptr, len); - } - - private Scanner() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index e51995f93..b2242a11c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -28,6 +28,8 @@ import java.util.LinkedList; import java.util.List; +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.internal.JGitText; @@ -38,6 +40,7 @@ import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; /** @@ -107,6 +110,8 @@ private static File getSymRef(File workTree, File dotGit, FS fs) private File workTree; + private String initialBranch = Constants.MASTER; + /** Directories limiting the search for a Git repository. */ private List ceilingDirectories; @@ -349,6 +354,43 @@ public File getIndexFile() { return indexFile; } + /** + * Set the initial branch of the new repository. If not specified + * ({@code null} or empty), fall back to the default name (currently + * master). + * + * @param branch + * initial branch name of the new repository. If {@code null} or + * empty the configured default branch will be used. + * @return {@code this} + * @throws InvalidRefNameException + * if the branch name is not valid + * + * @since 5.11 + */ + public B setInitialBranch(String branch) throws InvalidRefNameException { + if (StringUtils.isEmptyOrNull(branch)) { + this.initialBranch = Constants.MASTER; + } else { + if (!Repository.isValidRefName(Constants.R_HEADS + branch)) { + throw new InvalidRefNameException(MessageFormat + .format(JGitText.get().branchNameInvalid, branch)); + } + this.initialBranch = branch; + } + return self(); + } + + /** + * Get the initial branch of the new repository. + * + * @return the initial branch of the new repository. + * @since 5.11 + */ + public @NonNull String getInitialBranch() { + return initialBranch; + } + /** * Read standard Git environment variables and configure from those. *

    diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java index 4f93fda49..1665f051e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2007, Dave Watson - * Copyright (C) 2006-2007, Robin Rosenberg - * Copyright (C) 2006-2007, Shawn O. Pearce and others + * Copyright (C) 2006, 2007, Robin Rosenberg + * Copyright (C) 2006, 2020, Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -16,14 +16,11 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; -import java.text.MessageFormat; import java.util.List; -import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.util.References; /** @@ -37,7 +34,7 @@ * and obtain a {@link org.eclipse.jgit.revwalk.RevCommit} instance by calling * {@link org.eclipse.jgit.revwalk.RevWalk#parseCommit(AnyObjectId)}. */ -public class CommitBuilder { +public class CommitBuilder extends ObjectBuilder { private static final ObjectId[] EMPTY_OBJECTID_LIST = new ObjectId[0]; private static final byte[] htree = Constants.encodeASCII("tree"); //$NON-NLS-1$ @@ -50,28 +47,17 @@ public class CommitBuilder { private static final byte[] hgpgsig = Constants.encodeASCII("gpgsig"); //$NON-NLS-1$ - private static final byte[] hencoding = Constants.encodeASCII("encoding"); //$NON-NLS-1$ - private ObjectId treeId; private ObjectId[] parentIds; - private PersonIdent author; - private PersonIdent committer; - private GpgSignature gpgSignature; - - private String message; - - private Charset encoding; - /** * Initialize an empty commit. */ public CommitBuilder() { parentIds = EMPTY_OBJECTID_LIST; - encoding = UTF_8; } /** @@ -98,8 +84,9 @@ public void setTreeId(AnyObjectId id) { * * @return the author of this commit (who wrote it). */ + @Override public PersonIdent getAuthor() { - return author; + return super.getAuthor(); } /** @@ -108,8 +95,9 @@ public PersonIdent getAuthor() { * @param newAuthor * the new author. Should not be null. */ + @Override public void setAuthor(PersonIdent newAuthor) { - author = newAuthor; + super.setAuthor(newAuthor); } /** @@ -131,38 +119,6 @@ public void setCommitter(PersonIdent newCommitter) { committer = newCommitter; } - /** - * Set the GPG signature of this commit. - *

    - * Note, the signature set here will change the payload of the commit, i.e. - * the output of {@link #build()} will include the signature. Thus, the - * typical flow will be: - *

      - *
    1. call {@link #build()} without a signature set to obtain payload
      2. - *
    2. create {@link GpgSignature} from payload
      3. - *
    3. set {@link GpgSignature}
      4. - *
    - *

    - * - * @param newSignature - * the signature to set or null to unset - * @since 5.3 - */ - public void setGpgSignature(GpgSignature newSignature) { - gpgSignature = newSignature; - } - - /** - * Get the GPG signature of this commit. - * - * @return the GPG signature of this commit, maybe null if the - * commit is not to be signed - * @since 5.3 - */ - public GpgSignature getGpgSignature() { - return gpgSignature; - } - /** * Get the ancestors of this commit. * @@ -238,25 +194,6 @@ public void addParentId(AnyObjectId additionalParent) { } } - /** - * Get the complete commit message. - * - * @return the complete commit message. - */ - public String getMessage() { - return message; - } - - /** - * Set the commit message. - * - * @param newMessage - * the commit message. Should not be null. - */ - public void setMessage(String newMessage) { - message = newMessage; - } - /** * Set the encoding for the commit information. * @@ -267,37 +204,10 @@ public void setMessage(String newMessage) { */ @Deprecated public void setEncoding(String encodingName) { - encoding = Charset.forName(encodingName); + setEncoding(Charset.forName(encodingName)); } - /** - * Set the encoding for the commit information. - * - * @param enc - * the encoding to use. - */ - public void setEncoding(Charset enc) { - encoding = enc; - } - - /** - * Get the encoding that should be used for the commit message text. - * - * @return the encoding that should be used for the commit message text. - */ - public Charset getEncoding() { - return encoding; - } - - /** - * Format this builder's state as a commit object. - * - * @return this object in the canonical commit format, suitable for storage - * in a repository. - * @throws java.io.UnsupportedEncodingException - * the encoding specified by {@link #getEncoding()} is not - * supported by this Java runtime. - */ + @Override public byte[] build() throws UnsupportedEncodingException { ByteArrayOutputStream os = new ByteArrayOutputStream(); OutputStreamWriter w = new OutputStreamWriter(os, getEncoding()); @@ -326,19 +236,16 @@ public byte[] build() throws UnsupportedEncodingException { w.flush(); os.write('\n'); - if (getGpgSignature() != null) { + GpgSignature signature = getGpgSignature(); + if (signature != null) { os.write(hgpgsig); os.write(' '); - writeGpgSignatureString(getGpgSignature().toExternalString(), os); + writeMultiLineHeader(signature.toExternalString(), os, + true); os.write('\n'); } - if (!References.isSameObject(getEncoding(), UTF_8)) { - os.write(hencoding); - os.write(' '); - os.write(Constants.encodeASCII(getEncoding().name())); - os.write('\n'); - } + writeEncoding(getEncoding(), os); os.write('\n'); @@ -355,58 +262,6 @@ public byte[] build() throws UnsupportedEncodingException { return os.toByteArray(); } - /** - * Writes signature to output as per gpgsig - * header. - *

    - * CRLF and CR will be sanitized to LF and signature will have a hanging - * indent of one space starting with line two. A trailing line break is - * not written; the caller is supposed to terminate the GPG - * signature header by writing a single newline. - *

    - * - * @param in - * signature string with line breaks - * @param out - * output stream - * @throws IOException - * thrown by the output stream - * @throws IllegalArgumentException - * if the signature string contains non 7-bit ASCII chars - */ - static void writeGpgSignatureString(String in, OutputStream out) - throws IOException, IllegalArgumentException { - int length = in.length(); - for (int i = 0; i < length; ++i) { - char ch = in.charAt(i); - switch (ch) { - case '\r': - if (i + 1 < length && in.charAt(i + 1) == '\n') { - ++i; - } - if (i + 1 < length) { - out.write('\n'); - out.write(' '); - } - break; - case '\n': - if (i + 1 < length) { - out.write('\n'); - out.write(' '); - } - break; - default: - // sanity check - if (ch > 127) - throw new IllegalArgumentException(MessageFormat - .format(JGitText.get().notASCIIString, in)); - out.write(ch); - break; - } - } - } - /** * Format this builder's state as a commit object. * @@ -439,7 +294,7 @@ public String toString() { } r.append("author "); - r.append(author != null ? author.toString() : "NOT_SET"); + r.append(getAuthor() != null ? getAuthor().toString() : "NOT_SET"); r.append("\n"); r.append("committer "); @@ -447,17 +302,20 @@ public String toString() { r.append("\n"); r.append("gpgSignature "); - r.append(gpgSignature != null ? gpgSignature.toString() : "NOT_SET"); + GpgSignature signature = getGpgSignature(); + r.append(signature != null ? signature.toString() + : "NOT_SET"); r.append("\n"); - if (encoding != null && !References.isSameObject(encoding, UTF_8)) { + Charset encoding = getEncoding(); + if (!References.isSameObject(encoding, UTF_8)) { r.append("encoding "); r.append(encoding.name()); r.append("\n"); } r.append("\n"); - r.append(message != null ? message : ""); + r.append(getMessage() != null ? getMessage() : ""); r.append("}"); return r.toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 4fcf8e2dc..03c1ef904 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -92,14 +92,28 @@ public final class ConfigConstants { */ public static final String CONFIG_GPG_SECTION = "gpg"; + /** + * The "protocol" section + * @since 5.9 + */ + public static final String CONFIG_PROTOCOL_SECTION = "protocol"; + /** * The "format" key * @since 5.2 */ public static final String CONFIG_KEY_FORMAT = "format"; + /** + * The "program" key + * + * @since 5.11 + */ + public static final String CONFIG_KEY_PROGRAM = "program"; + /** * The "signingKey" key + * * @since 5.2 */ public static final String CONFIG_KEY_SIGNINGKEY = "signingKey"; @@ -110,14 +124,30 @@ public final class ConfigConstants { */ public static final String CONFIG_COMMIT_SECTION = "commit"; + /** + * The "tag" section + * + * @since 5.11 + */ + public static final String CONFIG_TAG_SECTION = "tag"; + /** * The "gpgSign" key + * * @since 5.2 */ public static final String CONFIG_KEY_GPGSIGN = "gpgSign"; + /** + * The "forceSignAnnotated" key + * + * @since 5.11 + */ + public static final String CONFIG_KEY_FORCE_SIGN_ANNOTATED = "forceSignAnnotated"; + /** * The "hooksPath" key. + * * @since 5.6 */ public static final String CONFIG_KEY_HOOKS_PATH = "hooksPath"; @@ -531,12 +561,6 @@ public final class ConfigConstants { */ public static final String CONFIG_REF_STORAGE_REFTABLE = "reftable"; - /** - * The "reftree" refStorage format - * @since 5.7 - */ - public static final String CONFIG_REFSTORAGE_REFTREE = "reftree"; - /** * The "jmx" section * @since 5.1.13 @@ -685,10 +709,23 @@ public final class ConfigConstants { public static final String CONFIG_INDEX_SECTION = "index"; /** - * The "index.version" key + * The "version" key * * @since 5.9 */ public static final String CONFIG_KEY_VERSION = "version"; + /** + * The "init" section + * + * @since 5.11 + */ + public static final String CONFIG_INIT_SECTION = "init"; + + /** + * The "defaultBranch" key + * + * @since 5.11 + */ + public static final String CONFIG_KEY_DEFAULT_BRANCH = "defaultbranch"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index c1527bc47..427a235f3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Salesforce. and others + * Copyright (C) 2018, 2021 Salesforce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -43,16 +43,65 @@ public String toConfigValue() { } } - private final Config config; + private final GpgFormat keyFormat; + + private final String signingKey; + + private final String program; + + private final boolean signCommits; + + private final boolean signAllTags; + + private final boolean forceAnnotated; /** - * Create a new GPG config, which will read configuration from config. + * Create a {@link GpgConfig} with the given parameters and default + * {@code true} for signing commits and {@code false} for tags. + * + * @param keySpec + * to use + * @param format + * to use + * @param gpgProgram + * to use + * @since 5.11 + */ + public GpgConfig(String keySpec, GpgFormat format, String gpgProgram) { + keyFormat = format; + signingKey = keySpec; + program = gpgProgram; + signCommits = true; + signAllTags = false; + forceAnnotated = false; + } + + /** + * Create a new GPG config that reads the configuration from config. * * @param config * the config to read from */ public GpgConfig(Config config) { - this.config = config; + keyFormat = config.getEnum(GpgFormat.values(), + ConfigConstants.CONFIG_GPG_SECTION, null, + ConfigConstants.CONFIG_KEY_FORMAT, GpgFormat.OPENPGP); + signingKey = config.getString(ConfigConstants.CONFIG_USER_SECTION, null, + ConfigConstants.CONFIG_KEY_SIGNINGKEY); + + String exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, + keyFormat.toConfigValue(), ConfigConstants.CONFIG_KEY_PROGRAM); + if (exe == null) { + exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, null, + ConfigConstants.CONFIG_KEY_PROGRAM); + } + program = exe; + signCommits = config.getBoolean(ConfigConstants.CONFIG_COMMIT_SECTION, + ConfigConstants.CONFIG_KEY_GPGSIGN, false); + signAllTags = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, + ConfigConstants.CONFIG_KEY_GPGSIGN, false); + forceAnnotated = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, + ConfigConstants.CONFIG_KEY_FORCE_SIGN_ANNOTATED, false); } /** @@ -61,9 +110,19 @@ public GpgConfig(Config config) { * @return the {@link org.eclipse.jgit.lib.GpgConfig.GpgFormat} */ public GpgFormat getKeyFormat() { - return config.getEnum(GpgFormat.values(), - ConfigConstants.CONFIG_GPG_SECTION, null, - ConfigConstants.CONFIG_KEY_FORMAT, GpgFormat.OPENPGP); + return keyFormat; + } + + /** + * Retrieves the value of the configured GPG program to use, as defined by + * gpg.openpgp.program, gpg.x509.program (depending on the defined + * {@link #getKeyFormat() format}), or gpg.program. + * + * @return the program string configured, or {@code null} if none + * @since 5.11 + */ + public String getProgram() { + return program; } /** @@ -72,8 +131,7 @@ public GpgFormat getKeyFormat() { * @return the value of user.signingKey (may be null) */ public String getSigningKey() { - return config.getString(ConfigConstants.CONFIG_USER_SECTION, null, - ConfigConstants.CONFIG_KEY_SIGNINGKEY); + return signingKey; } /** @@ -82,7 +140,29 @@ public String getSigningKey() { * @return the value of commit.gpgSign (defaults to false) */ public boolean isSignCommits() { - return config.getBoolean(ConfigConstants.CONFIG_COMMIT_SECTION, - ConfigConstants.CONFIG_KEY_GPGSIGN, false); + return signCommits; + } + + /** + * Retrieves the value of git config {@code tag.gpgSign}. + * + * @return the value of {@code tag.gpgSign}; by default {@code false} + * + * @since 5.11 + */ + public boolean isSignAllTags() { + return signAllTags; + } + + /** + * Retrieves the value of git config {@code tag.forceSignAnnotated}. + * + * @return the value of {@code tag.forceSignAnnotated}; by default + * {@code false} + * + * @since 5.11 + */ + public boolean isSignAnnotated() { + return forceAnnotated; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java new file mode 100644 index 000000000..074f46567 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; +import org.eclipse.jgit.transport.CredentialsProvider; + +/** + * Creates GPG signatures for Git objects. + * + * @since 5.11 + */ +public interface GpgObjectSigner { + + /** + * Signs the specified object. + * + *

    + * Implementors should obtain the payload for signing from the specified + * object via {@link ObjectBuilder#build()} and create a proper + * {@link GpgSignature}. The generated signature must be set on the + * specified {@code object} (see + * {@link ObjectBuilder#setGpgSignature(GpgSignature)}). + *

    + *

    + * Any existing signature on the object must be discarded prior obtaining + * the payload via {@link ObjectBuilder#build()}. + *

    + * + * @param object + * the object to sign (must not be {@code null} and must be + * complete to allow proper calculation of payload) + * @param gpgSigningKey + * the signing key to locate (passed as is to the GPG signing + * tool as is; eg., value of user.signingkey) + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @param config + * GPG settings from the git config + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + void signObject(@NonNull ObjectBuilder object, + @Nullable String gpgSigningKey, @NonNull PersonIdent committer, + CredentialsProvider credentialsProvider, GpgConfig config) + throws CanceledException, UnsupportedSigningFormatException; + + /** + * Indicates if a signing key is available for the specified committer + * and/or signing key. + * + * @param gpgSigningKey + * the signing key to locate (passed as is to the GPG signing + * tool as is; eg., value of user.signingkey) + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @param config + * GPG settings from the git config + * @return true if a signing key is available, + * false otherwise + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + public abstract boolean canLocateSigningKey(@Nullable String gpgSigningKey, + @NonNull PersonIdent committer, + CredentialsProvider credentialsProvider, GpgConfig config) + throws CanceledException, UnsupportedSigningFormatException; + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java new file mode 100644 index 000000000..a7a39c998 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.util.Date; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.revwalk.RevObject; + +/** + * A {@code GpgVerifier} can verify GPG signatures on git commits and tags. + * + * @since 5.11 + */ +public interface GpgSignatureVerifier { + + /** + * Verifies the signature on a signed commit or tag. + * + * @param object + * to verify + * @param config + * the {@link GpgConfig} to use + * @return a {@link SignatureVerification} describing the outcome of the + * verification, or {@code null} if the object was not signed + * @throws IOException + * if an error occurs getting a public key + * @throws org.eclipse.jgit.api.errors.JGitInternalException + * if signature verification fails + */ + @Nullable + SignatureVerification verifySignature(@NonNull RevObject object, + @NonNull GpgConfig config) throws IOException; + + + /** + * Verifies a given signature for given data. + * + * @param data + * the signature is for + * @param signatureData + * the ASCII-armored signature + * @return a {@link SignatureVerification} describing the outcome + * @throws IOException + * if the signature cannot be parsed + * @throws JGitInternalException + * if signature verification fails + */ + public SignatureVerification verify(byte[] data, byte[] signatureData) + throws IOException; + + /** + * Retrieves the name of this verifier. This should be a short string + * identifying the engine that verified the signature, like "gpg" if GPG is + * used, or "bc" for a BouncyCastle implementation. + * + * @return the name + */ + @NonNull + String getName(); + + /** + * A {@link GpgSignatureVerifier} may cache public keys to speed up + * verifying signatures on multiple objects. This clears this cache, if any. + */ + void clear(); + + /** + * A {@code SignatureVerification} returns data about a (positively or + * negatively) verified signature. + */ + interface SignatureVerification { + + // Data about the signature. + + @NonNull + Date getCreationDate(); + + // Data from the signature used to find a public key. + + /** + * Obtains the signer as stored in the signature, if known. + * + * @return the signer, or {@code null} if unknown + */ + String getSigner(); + + /** + * Obtains the short or long fingerprint of the public key as stored in + * the signature, if known. + * + * @return the fingerprint, or {@code null} if unknown + */ + String getKeyFingerprint(); + + // Some information about the found public key. + + /** + * Obtains the OpenPGP user ID associated with the key. + * + * @return the user id, or {@code null} if unknown + */ + String getKeyUser(); + + /** + * Tells whether the public key used for this signature verification was + * expired when the signature was created. + * + * @return {@code true} if the key was expired already, {@code false} + * otherwise + */ + boolean isExpired(); + + /** + * Obtains the trust level of the public key used to verify the + * signature. + * + * @return the trust level + */ + @NonNull + TrustLevel getTrustLevel(); + + // The verification result. + + /** + * Tells whether the signature verification was successful. + * + * @return {@code true} if the signature was verified successfully; + * {@code false} if not. + */ + boolean getVerified(); + + /** + * Obtains a human-readable message giving additional information about + * the outcome of the verification. + * + * @return the message, or {@code null} if none set. + */ + String getMessage(); + } + + /** + * The owner's trust in a public key. + */ + enum TrustLevel { + UNKNOWN, NEVER, MARGINAL, FULL, ULTIMATE + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java new file mode 100644 index 000000000..4b1dbedeb --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.util.Iterator; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A {@code GpgSignatureVerifierFactory} creates {@link GpgSignatureVerifier} instances. + * + * @since 5.11 + */ +public abstract class GpgSignatureVerifierFactory { + + private static final Logger LOG = LoggerFactory + .getLogger(GpgSignatureVerifierFactory.class); + + private static volatile GpgSignatureVerifierFactory defaultFactory = loadDefault(); + + private static GpgSignatureVerifierFactory loadDefault() { + try { + ServiceLoader loader = ServiceLoader + .load(GpgSignatureVerifierFactory.class); + Iterator iter = loader.iterator(); + if (iter.hasNext()) { + return iter.next(); + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return null; + } + + /** + * Retrieves the default factory. + * + * @return the default factory or {@code null} if none set + */ + public static GpgSignatureVerifierFactory getDefault() { + return defaultFactory; + } + + /** + * Sets the default factory. + * + * @param factory + * the new default factory + */ + public static void setDefault(GpgSignatureVerifierFactory factory) { + defaultFactory = factory; + } + + /** + * Creates a new {@link GpgSignatureVerifier}. + * + * @return the new {@link GpgSignatureVerifier} + */ + public abstract GpgSignatureVerifier getVerifier(); + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectBuilder.java new file mode 100644 index 000000000..4b7054f72 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectBuilder.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2020, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Objects; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.util.References; + +/** + * Common base class for {@link CommitBuilder} and {@link TagBuilder}. + * + * @since 5.11 + */ +public abstract class ObjectBuilder { + + /** Byte representation of "encoding". */ + private static final byte[] hencoding = Constants.encodeASCII("encoding"); //$NON-NLS-1$ + + private PersonIdent author; + + private GpgSignature gpgSignature; + + private String message; + + private Charset encoding = StandardCharsets.UTF_8; + + /** + * Retrieves the author of this object. + * + * @return the author of this object, or {@code null} if not set yet + */ + protected PersonIdent getAuthor() { + return author; + } + + /** + * Sets the author (name, email address, and date) of this object. + * + * @param newAuthor + * the new author, must be non-{@code null} + */ + protected void setAuthor(PersonIdent newAuthor) { + author = Objects.requireNonNull(newAuthor); + } + + /** + * Sets the GPG signature of this object. + *

    + * Note, the signature set here will change the payload of the object, i.e. + * the output of {@link #build()} will include the signature. Thus, the + * typical flow will be: + *

      + *
    1. call {@link #build()} without a signature set to obtain payload
      2. + *
    2. create {@link GpgSignature} from payload
      3. + *
    3. set {@link GpgSignature}
      4. + *
    + *

    + * + * @param gpgSignature + * the signature to set or {@code null} to unset + * @since 5.3 + */ + public void setGpgSignature(@Nullable GpgSignature gpgSignature) { + this.gpgSignature = gpgSignature; + } + + /** + * Retrieves the GPG signature of this object. + * + * @return the GPG signature of this object, or {@code null} if the object + * is not signed + * @since 5.3 + */ + @Nullable + public GpgSignature getGpgSignature() { + return gpgSignature; + } + + /** + * Retrieves the complete message of the object. + * + * @return the complete message; can be {@code null}. + */ + @Nullable + public String getMessage() { + return message; + } + + /** + * Sets the message (commit message, or message of an annotated tag). + * + * @param message + * the message. + */ + public void setMessage(@Nullable String message) { + this.message = message; + } + + /** + * Retrieves the encoding that should be used for the message text. + * + * @return the encoding that should be used for the message text. + */ + @NonNull + public Charset getEncoding() { + return encoding; + } + + /** + * Sets the encoding for the object message. + * + * @param encoding + * the encoding to use. + */ + public void setEncoding(@NonNull Charset encoding) { + this.encoding = encoding; + } + + /** + * Format this builder's state as a git object. + * + * @return this object in the canonical git format, suitable for storage in + * a repository. + * @throws java.io.UnsupportedEncodingException + * the encoding specified by {@link #getEncoding()} is not + * supported by this Java runtime. + */ + @NonNull + public abstract byte[] build() throws UnsupportedEncodingException; + + /** + * Writes signature to output as per gpgsig + * header. + *

    + * CRLF and CR will be sanitized to LF and signature will have a hanging + * indent of one space starting with line two. A trailing line break is + * not written; the caller is supposed to terminate the GPG + * signature header by writing a single newline. + *

    + * + * @param in + * signature string with line breaks + * @param out + * output stream + * @param enforceAscii + * whether to throw {@link IllegalArgumentException} if non-ASCII + * characters are encountered + * @throws IOException + * thrown by the output stream + * @throws IllegalArgumentException + * if the signature string contains non 7-bit ASCII chars and + * {@code enforceAscii == true} + */ + static void writeMultiLineHeader(@NonNull String in, + @NonNull OutputStream out, boolean enforceAscii) + throws IOException, IllegalArgumentException { + int length = in.length(); + for (int i = 0; i < length; ++i) { + char ch = in.charAt(i); + switch (ch) { + case '\r': + if (i + 1 < length && in.charAt(i + 1) == '\n') { + ++i; + } + if (i + 1 < length) { + out.write('\n'); + out.write(' '); + } + break; + case '\n': + if (i + 1 < length) { + out.write('\n'); + out.write(' '); + } + break; + default: + // sanity check + if (ch > 127 && enforceAscii) + throw new IllegalArgumentException(MessageFormat + .format(JGitText.get().notASCIIString, in)); + out.write(ch); + break; + } + } + } + + /** + * Writes an "encoding" header. + * + * @param encoding + * to write + * @param out + * to write to + * @throws IOException + * if writing fails + */ + static void writeEncoding(@NonNull Charset encoding, + @NonNull OutputStream out) throws IOException { + if (!References.isSameObject(encoding, UTF_8)) { + out.write(hencoding); + out.write(' '); + out.write(Constants.encodeASCII(encoding.name())); + out.write('\n'); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java index 6bb6ae590..718ed8914 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java @@ -17,9 +17,18 @@ import java.util.List; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.revwalk.BitmappedObjectReachabilityChecker; +import org.eclipse.jgit.internal.revwalk.BitmappedReachabilityChecker; +import org.eclipse.jgit.internal.revwalk.PedestrianObjectReachabilityChecker; +import org.eclipse.jgit.internal.revwalk.PedestrianReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevWalk; /** * Reads an {@link org.eclipse.jgit.lib.ObjectDatabase} for a single thread. @@ -407,6 +416,54 @@ public BitmapIndex getBitmapIndex() throws IOException { return null; } + /** + * Create a reachability checker that will use bitmaps if possible. + * + * @param rw + * revwalk for use by the reachability checker + * @return the most efficient reachability checker for this repository. + * @throws IOException + * if it cannot open any of the underlying indices. + * + * @since 5.11 + */ + @NonNull + public ReachabilityChecker createReachabilityChecker(RevWalk rw) + throws IOException { + if (getBitmapIndex() != null) { + return new BitmappedReachabilityChecker(rw); + } + + return new PedestrianReachabilityChecker(true, rw); + } + + /** + * Create an object reachability checker that will use bitmaps if possible. + * + * This reachability checker accepts any object as target. For checks + * exclusively between commits, use + * {@link #createReachabilityChecker(RevWalk)}. + * + * @param ow + * objectwalk for use by the reachability checker + * @return the most efficient object reachability checker for this + * repository. + * + * @throws IOException + * if it cannot open any of the underlying indices. + * + * @since 5.11 + */ + @NonNull + public ObjectReachabilityChecker createObjectReachabilityChecker( + ObjectWalk ow) throws IOException { + if (getBitmapIndex() != null) { + return new BitmappedObjectReachabilityChecker(ow); + } + + return new PedestrianObjectReachabilityChecker(ow); + } + /** * Get the {@link org.eclipse.jgit.lib.ObjectInserter} from which this * reader was created using {@code inserter.newReader()} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 6832c9cd8..7b7bdebac 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -21,6 +21,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; @@ -413,6 +416,31 @@ public List getRefsByPrefix(String prefix) throws IOException { return Collections.unmodifiableList(result); } + /** + * Returns refs whose names start with a given prefix excluding all refs that + * start with one of the given prefixes. + * + *

    + * The default implementation is not efficient. Implementors of {@link RefDatabase} + * should override this method directly if a better implementation is possible. + * + * @param include string that names of refs should start with; may be empty. + * @param excludes strings that names of refs can't start with; may be empty. + * @return immutable list of refs whose names start with {@code prefix} and none + * of the strings in {@code exclude}. + * @throws java.io.IOException the reference space cannot be accessed. + * @since 5.11 + */ + @NonNull + public List getRefsByPrefixWithExclusions(String include, Set excludes) + throws IOException { + Stream refs = getRefs(include).values().stream(); + for(String exclude: excludes) { + refs = refs.filter(r -> !r.getName().startsWith(exclude)); + } + return Collections.unmodifiableList(refs.collect(Collectors.toList())); + } + /** * Returns refs whose names start with one of the given prefixes. *

    diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index a7a832c1a..1e8a6c917 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -127,6 +127,8 @@ public static ListenerList getGlobalListenerList() { /** If not bare, the index file caching the working file states. */ private final File indexFile; + private final String initialBranch; + /** * Initialize a new repository instance. * @@ -138,6 +140,7 @@ protected Repository(BaseRepositoryBuilder options) { fs = options.getFS(); workTree = options.getWorkTree(); indexFile = options.getIndexFile(); + initialBranch = options.getInitialBranch(); } /** @@ -1033,6 +1036,16 @@ public String getBranch() throws IOException { return null; } + /** + * Get the initial branch name of a new repository + * + * @return the initial branch name of a new repository + * @since 5.11 + */ + protected @NonNull String getInitialBranch() { + return initialBranch; + } + /** * Objects known to exist but not expressed by {@link #getAllRefs()}. *

    diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java index d36ccd5be..41f291b57 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -193,7 +194,7 @@ static void reconfigure(RepositoryCacheConfig repositoryCacheConfig) { cache.configureEviction(repositoryCacheConfig); } - private final ConcurrentHashMap cacheMap; + private final Map cacheMap; private final Lock[] openLocks; @@ -201,6 +202,8 @@ static void reconfigure(RepositoryCacheConfig repositoryCacheConfig) { private volatile long expireAfter; + private final Object schedulerLock = new Lock(); + private RepositoryCache() { cacheMap = new ConcurrentHashMap<>(); openLocks = new Lock[4]; @@ -214,7 +217,7 @@ private void configureEviction( RepositoryCacheConfig repositoryCacheConfig) { expireAfter = repositoryCacheConfig.getExpireAfter(); ScheduledThreadPoolExecutor scheduler = WorkQueue.getExecutor(); - synchronized (scheduler) { + synchronized (schedulerLock) { if (cleanupTask != null) { cleanupTask.cancel(false); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java index 71f01150c..facb4a54b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2006-2008, Robin Rosenberg + * Copyright (C) 2006, 2008, Robin Rosenberg * Copyright (C) 2008, Shawn O. Pearce - * Copyright (C) 2010, Chris Aniszczyk and others + * Copyright (C) 2010, 2020, Chris Aniszczyk and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -17,8 +17,13 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.util.References; /** * Mutable builder to construct an annotated tag recording a project state. @@ -30,17 +35,22 @@ * and obtain a {@link org.eclipse.jgit.revwalk.RevTag} instance by calling * {@link org.eclipse.jgit.revwalk.RevWalk#parseTag(AnyObjectId)}. */ -public class TagBuilder { +public class TagBuilder extends ObjectBuilder { + + private static final byte[] hobject = Constants.encodeASCII("object"); //$NON-NLS-1$ + + private static final byte[] htype = Constants.encodeASCII("type"); //$NON-NLS-1$ + + private static final byte[] htag = Constants.encodeASCII("tag"); //$NON-NLS-1$ + + private static final byte[] htagger = Constants.encodeASCII("tagger"); //$NON-NLS-1$ + private ObjectId object; private int type = Constants.OBJ_BAD; private String tag; - private PersonIdent tagger; - - private String message; - /** * Get the type of object this tag refers to. * @@ -109,7 +119,7 @@ public void setTag(String shortName) { * @return creator of this tag. May be null. */ public PersonIdent getTagger() { - return tagger; + return getAuthor(); } /** @@ -119,26 +129,7 @@ public PersonIdent getTagger() { * the creator. May be null. */ public void setTagger(PersonIdent taggerIdent) { - tagger = taggerIdent; - } - - /** - * Get the complete commit message. - * - * @return the complete commit message. - */ - public String getMessage() { - return message; - } - - /** - * Set the tag's message. - * - * @param newMessage - * the tag's message. - */ - public void setMessage(String newMessage) { - message = newMessage; + setAuthor(taggerIdent); } /** @@ -147,31 +138,65 @@ public void setMessage(String newMessage) { * @return this object in the canonical annotated tag format, suitable for * storage in a repository. */ - public byte[] build() { + @Override + public byte[] build() throws UnsupportedEncodingException { ByteArrayOutputStream os = new ByteArrayOutputStream(); try (OutputStreamWriter w = new OutputStreamWriter(os, - UTF_8)) { - w.write("object "); //$NON-NLS-1$ - getObjectId().copyTo(w); - w.write('\n'); + getEncoding())) { - w.write("type "); //$NON-NLS-1$ - w.write(Constants.typeString(getObjectType())); - w.write("\n"); //$NON-NLS-1$ + os.write(hobject); + os.write(' '); + getObjectId().copyTo(os); + os.write('\n'); - w.write("tag "); //$NON-NLS-1$ + os.write(htype); + os.write(' '); + os.write(Constants + .encodeASCII(Constants.typeString(getObjectType()))); + os.write('\n'); + + os.write(htag); + os.write(' '); w.write(getTag()); - w.write("\n"); //$NON-NLS-1$ + w.flush(); + os.write('\n'); if (getTagger() != null) { - w.write("tagger "); //$NON-NLS-1$ + os.write(htagger); + os.write(' '); w.write(getTagger().toExternalString()); - w.write('\n'); + w.flush(); + os.write('\n'); } - w.write('\n'); - if (getMessage() != null) - w.write(getMessage()); + writeEncoding(getEncoding(), os); + + os.write('\n'); + String msg = getMessage(); + if (msg != null) { + w.write(msg); + w.flush(); + } + + GpgSignature signature = getGpgSignature(); + if (signature != null) { + if (msg != null && !msg.isEmpty() && !msg.endsWith("\n")) { //$NON-NLS-1$ + // If signed, the message *must* end with a linefeed + // character, otherwise signature verification will fail. + // (The signature will have been computed over the payload + // containing the message without LF, but will be verified + // against a payload with the LF.) The signature must start + // on a new line. + throw new JGitInternalException( + JGitText.get().signedTagMessageNoLf); + } + String externalForm = signature.toExternalString(); + w.write(externalForm); + w.flush(); + if (!externalForm.endsWith("\n")) { //$NON-NLS-1$ + os.write('\n'); + } + } } catch (IOException err) { // This should never occur, the only way to get it above is // for the ByteArrayOutputStream to throw, but it doesn't. @@ -185,10 +210,17 @@ public byte[] build() { * Format this builder's state as an annotated tag object. * * @return this object in the canonical annotated tag format, suitable for - * storage in a repository. + * storage in a repository, or {@code null} if the tag cannot be + * encoded + * @deprecated since 5.11; use {@link #build()} instead */ + @Deprecated public byte[] toByteArray() { - return build(); + try { + return build(); + } catch (UnsupportedEncodingException e) { + return null; + } } /** {@inheritDoc} */ @@ -211,14 +243,23 @@ public String toString() { r.append(tag != null ? tag : "NOT_SET"); r.append("\n"); - if (tagger != null) { + if (getTagger() != null) { r.append("tagger "); - r.append(tagger); + r.append(getTagger()); + r.append("\n"); + } + + Charset encoding = getEncoding(); + if (!References.isSameObject(encoding, UTF_8)) { + r.append("encoding "); + r.append(encoding.name()); r.append("\n"); } r.append("\n"); - r.append(message != null ? message : ""); + r.append(getMessage() != null ? getMessage() : ""); + GpgSignature signature = getGpgSignature(); + r.append(signature != null ? signature.toExternalString() : ""); r.append("}"); return r.toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 6c217fdf2..4bfb38d28 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -703,18 +703,21 @@ protected boolean processEntry(CanonicalTreeParser base, // conflict between ours and theirs. file/folder conflicts between // base/index/workingTree and something else are not relevant or // detected later - if (nonTree(modeO) && !nonTree(modeT)) { + if (nonTree(modeO) != nonTree(modeT)) { + if (ignoreConflicts) { + // In case of merge failures, ignore this path instead of reporting unmerged, so + // a caller can use virtual commit. This will not result in files with conflict + // markers in the index/working tree. The actual diff on the path will be + // computed directly on children. + enterSubtree = false; + return true; + } if (nonTree(modeB)) add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); - unmergedPaths.add(tw.getPathString()); - enterSubtree = false; - return true; - } - if (nonTree(modeT) && !nonTree(modeO)) { - if (nonTree(modeB)) - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); - add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); + if (nonTree(modeO)) + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); + if (nonTree(modeT)) + add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); unmergedPaths.add(tw.getPathString()); enterSubtree = false; return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java b/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java index d7dd3bee5..881873de6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.nls; import java.util.Locale; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.eclipse.jgit.errors.TranslationBundleLoadingException; @@ -110,7 +111,8 @@ public static void clear() { } private final Locale locale; - private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + + private final Map map = new ConcurrentHashMap<>(); private NLS(Locale locale) { this.locale = locale; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java index b875be927..0cabf0705 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java @@ -93,7 +93,7 @@ else if (t > when) head = n; } else { Entry p = q.next; - while (p != null && p.commit.commitTime > when) { + while (p != null && p.commit.commitTime >= when) { q = p; p = q.next; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java index 4c7a6f556..e6f9580bf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java @@ -172,14 +172,14 @@ public ObjectWalk(ObjectReader or) { * when the index fails to load. * * @since 5.8 + * @deprecated use + * {@code ObjectReader#createObjectReachabilityChecker(ObjectWalk)} + * instead. */ - public ObjectReachabilityChecker createObjectReachabilityChecker() + @Deprecated + public final ObjectReachabilityChecker createObjectReachabilityChecker() throws IOException { - if (reader.getBitmapIndex() != null) { - return new BitmappedObjectReachabilityChecker(this); - } - - return new PedestrianObjectReachabilityChecker(this); + return reader.createObjectReachabilityChecker(this); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java index cac257199..b9d145008 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2009, Google Inc. + * Copyright (C) 2008, 2009, Google Inc. * Copyright (C) 2008, Marek Zawirski - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2021, Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -18,7 +18,9 @@ import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.util.Arrays; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; @@ -35,6 +37,10 @@ * An annotated tag. */ public class RevTag extends RevObject { + + private static final byte[] hSignature = Constants + .encodeASCII("-----BEGIN PGP SIGNATURE-----"); //$NON-NLS-1$ + /** * Parse an annotated tag from its canonical format. * @@ -171,6 +177,71 @@ public final PersonIdent getTaggerIdent() { return RawParseUtils.parsePersonIdent(raw, nameB); } + private static int nextStart(byte[] prefix, byte[] buffer, int from) { + int stop = buffer.length - prefix.length + 1; + int ptr = from; + if (ptr > 0) { + ptr = RawParseUtils.nextLF(buffer, ptr - 1); + } + while (ptr < stop) { + int lineStart = ptr; + boolean found = true; + for (byte element : prefix) { + if (element != buffer[ptr++]) { + found = false; + break; + } + } + if (found) { + return lineStart; + } + do { + ptr = RawParseUtils.nextLF(buffer, ptr); + } while (ptr < stop && buffer[ptr] == '\n'); + } + return -1; + } + + private int getSignatureStart() { + byte[] raw = buffer; + int msgB = RawParseUtils.tagMessage(raw, 0); + if (msgB < 0) { + return msgB; + } + // Find the last signature start and return the rest + int start = nextStart(hSignature, raw, msgB); + if (start < 0) { + return start; + } + int next = RawParseUtils.nextLF(raw, start); + while (next < raw.length) { + int newStart = nextStart(hSignature, raw, next); + if (newStart < 0) { + break; + } + start = newStart; + next = RawParseUtils.nextLF(raw, start); + } + return start; + } + + /** + * Parse the GPG signature from the raw buffer. + * + * @return contents of the GPG signature; {@code null} if the tag was not + * signed. + * @since 5.11 + */ + @Nullable + public final byte[] getRawGpgSignature() { + byte[] raw = buffer; + int start = getSignatureStart(); + if (start < 0) { + return null; + } + return Arrays.copyOfRange(raw, start, raw.length); + } + /** * Parse the complete tag message and decode it to a string. *

    @@ -187,7 +258,12 @@ public final String getFullMessage() { if (msgB < 0) { return ""; //$NON-NLS-1$ } - return RawParseUtils.decode(guessEncoding(), raw, msgB, raw.length); + int signatureStart = getSignatureStart(); + int end = signatureStart < 0 ? raw.length : signatureStart; + if (end == msgB) { + return ""; //$NON-NLS-1$ + } + return RawParseUtils.decode(guessEncoding(), raw, msgB, end); } /** @@ -213,6 +289,16 @@ public final String getShortMessage() { } int msgE = RawParseUtils.endOfParagraph(raw, msgB); + int signatureStart = getSignatureStart(); + if (signatureStart >= msgB && msgE > signatureStart) { + msgE = signatureStart; + if (msgE > msgB) { + msgE--; + } + if (msgB == msgE) { + return ""; //$NON-NLS-1$ + } + } String str = RawParseUtils.decode(guessEncoding(), raw, msgB, msgE); if (RevCommit.hasLF(raw, msgB, msgE)) { str = StringUtils.replaceLineBreaksWithSpace(str); @@ -257,6 +343,22 @@ public final String getTagName() { return tagName; } + /** + * Obtain the raw unparsed tag body (NOTE - THIS IS NOT A COPY). + *

    + * This method is exposed only to provide very fast, efficient access to + * this tag's message buffer. Applications relying on this buffer should be + * very careful to ensure they do not modify its contents during their use + * of it. + * + * @return the raw unparsed tag body. This is NOT A COPY. Do not + * alter the returned array. + * @since 5.11 + */ + public final byte[] getRawBuffer() { + return buffer; + } + /** * Discard the message buffer to reduce memory usage. *

    diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index 6b62fcdf6..631d861c0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -236,13 +236,13 @@ public ObjectReader getObjectReader() { * if it cannot open any of the underlying indices. * * @since 5.4 + * @deprecated use {@code ObjectReader#createReachabilityChecker(RevWalk)} + * instead. */ - public ReachabilityChecker createReachabilityChecker() throws IOException { - if (reader.getBitmapIndex() != null) { - return new BitmappedReachabilityChecker(this); - } - - return new PedestrianReachabilityChecker(true, this); + @Deprecated + public final ReachabilityChecker createReachabilityChecker() + throws IOException { + return reader.createReachabilityChecker(this); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java index 3a3639862..3826bf740 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Marek Zawirski * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,7 +13,12 @@ package org.eclipse.jgit.transport; +import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_1; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2; import java.io.EOFException; import java.io.IOException; @@ -22,23 +27,29 @@ import java.text.MessageFormat; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.RemoteRepositoryException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.SymbolicRef; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.io.InterruptTimer; import org.eclipse.jgit.util.io.TimeoutInputStream; import org.eclipse.jgit.util.io.TimeoutOutputStream; @@ -92,17 +103,27 @@ abstract class BasePackConnection extends BaseConnection { protected boolean statelessRPC; /** Capability tokens advertised by the remote side. */ - private final Set remoteCapablities = new HashSet<>(); + private final Map remoteCapabilities = new HashMap<>(); /** Extra objects the remote has, but which aren't offered as refs. */ protected final Set additionalHaves = new HashSet<>(); + private TransferConfig.ProtocolVersion protocol = TransferConfig.ProtocolVersion.V0; + BasePackConnection(PackTransport packTransport) { transport = (Transport) packTransport; local = transport.local; uri = transport.uri; } + TransferConfig.ProtocolVersion getProtocolVersion() { + return protocol; + } + + void setProtocolVersion(@NonNull TransferConfig.ProtocolVersion protocol) { + this.protocol = protocol; + } + /** * Configure this connection with the directional pipes. * @@ -147,12 +168,15 @@ protected final void init(InputStream myIn, OutputStream myOut) { * {@link #close()} and the exception is wrapped (if necessary) and thrown * as a {@link org.eclipse.jgit.errors.TransportException}. * + * @return {@code true} if the refs were read; {@code false} otherwise + * indicating that {@link #lsRefs} must be called + * * @throws org.eclipse.jgit.errors.TransportException * the reference list could not be scanned. */ - protected void readAdvertisedRefs() throws TransportException { + protected boolean readAdvertisedRefs() throws TransportException { try { - readAdvertisedRefsImpl(); + return readAdvertisedRefsImpl(); } catch (TransportException err) { close(); throw err; @@ -162,35 +186,79 @@ protected void readAdvertisedRefs() throws TransportException { } } - private void readAdvertisedRefsImpl() throws IOException { - final LinkedHashMap avail = new LinkedHashMap<>(); - for (;;) { + private String readLine() throws IOException { + String line = pckIn.readString(); + if (PacketLineIn.isEnd(line)) { + return null; + } + if (line.startsWith("ERR ")) { //$NON-NLS-1$ + // This is a customized remote service error. + // Users should be informed about it. + throw new RemoteRepositoryException(uri, line.substring(4)); + } + return line; + } + + private boolean readAdvertisedRefsImpl() throws IOException { + final Map avail = new LinkedHashMap<>(); + final Map symRefs = new LinkedHashMap<>(); + for (boolean first = true;; first = false) { String line; - try { - line = pckIn.readString(); - } catch (EOFException eof) { - if (avail.isEmpty()) - throw noRepository(); - throw eof; - } - if (PacketLineIn.isEnd(line)) - break; - - if (line.startsWith("ERR ")) { //$NON-NLS-1$ - // This is a customized remote service error. - // Users should be informed about it. - throw new RemoteRepositoryException(uri, line.substring(4)); - } - - if (avail.isEmpty()) { + if (first) { + boolean isV1 = false; + try { + line = readLine(); + } catch (EOFException e) { + TransportException noRepo = noRepository(); + noRepo.initCause(e); + throw noRepo; + } + if (line != null && VERSION_1.equals(line)) { + // Same as V0, except for this extra line. We shouldn't get + // it since we never request V1. + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + isV1 = true; + line = readLine(); + } + if (line == null) { + break; + } final int nul = line.indexOf('\0'); if (nul >= 0) { - // The first line (if any) may contain "hidden" - // capability values after a NUL byte. - remoteCapablities.addAll( - Arrays.asList(line.substring(nul + 1).split(" "))); //$NON-NLS-1$ + // Protocol V0: The first line (if any) may contain + // "hidden" capability values after a NUL byte. + for (String capability : line.substring(nul + 1) + .split(" ")) { //$NON-NLS-1$ + if (capability.startsWith(CAPABILITY_SYMREF_PREFIX)) { + String[] parts = capability + .substring( + CAPABILITY_SYMREF_PREFIX.length()) + .split(":", 2); //$NON-NLS-1$ + if (parts.length == 2) { + symRefs.put(parts[0], parts[1]); + } + } else { + addCapability(capability); + } + } line = line.substring(0, nul); + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + } else if (!isV1 && VERSION_2.equals(line)) { + // Protocol V2: remaining lines are capabilities as + // key=value pairs + setProtocolVersion(TransferConfig.ProtocolVersion.V2); + readCapabilitiesV2(); + // Break out here so that stateless RPC transports get a + // chance to set up the output stream. + return false; + } else { + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + } + } else { + line = readLine(); + if (line == null) { + break; } } @@ -199,73 +267,214 @@ private void readAdvertisedRefsImpl() throws IOException { throw invalidRefAdvertisementLine(line); } String name = line.substring(41, line.length()); - if (avail.isEmpty() && name.equals("capabilities^{}")) { //$NON-NLS-1$ - // special line from git-receive-pack to show + if (first && name.equals("capabilities^{}")) { //$NON-NLS-1$ + // special line from git-receive-pack (protocol V0) to show // capabilities when there are no refs to advertise continue; } - final ObjectId id; - try { - id = ObjectId.fromString(line.substring(0, 40)); - } catch (InvalidObjectIdException e) { - PackProtocolException ppe = invalidRefAdvertisementLine(line); - ppe.initCause(e); - throw ppe; - } + final ObjectId id = toId(line, line.substring(0, 40)); if (name.equals(".have")) { //$NON-NLS-1$ additionalHaves.add(id); - } else if (name.endsWith("^{}")) { //$NON-NLS-1$ - name = name.substring(0, name.length() - 3); - final Ref prior = avail.get(name); - if (prior == null) - throw new PackProtocolException(uri, MessageFormat.format( - JGitText.get().advertisementCameBefore, name, name)); - - if (prior.getPeeledObjectId() != null) - throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$ - - avail.put(name, new ObjectIdRef.PeeledTag( - Ref.Storage.NETWORK, name, prior.getObjectId(), id)); } else { - final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag( - Ref.Storage.NETWORK, name, id)); - if (prior != null) - throw duplicateAdvertisement(name); + processLineV1(name, id, avail); } } - updateWithSymRefs(avail, extractSymRefsFromCapabilities(remoteCapablities)); + updateWithSymRefs(avail, symRefs); available(avail); + return true; } /** - * Finds values in the given capabilities of the form: + * Issue a protocol V2 ls-refs command and read its response. * - * 

    -	 * symref=source:target
-	 *
    - * - * And returns a Map of source->target entries. - * - * @param capabilities - * the capabilities lines - * @return a Map of the symref entries from capabilities - * @throws NullPointerException - * if capabilities, or any entry in it, is null + * @param refSpecs + * to produce ref prefixes from if the server supports git + * protocol V2 + * @param additionalPatterns + * to use for ref prefixes if the server supports git protocol V2 + * @throws TransportException + * if the command could not be run or its output not be read */ - static Map extractSymRefsFromCapabilities(Collection capabilities) { + protected void lsRefs(Collection refSpecs, + String... additionalPatterns) throws TransportException { + try { + lsRefsImpl(refSpecs, additionalPatterns); + } catch (TransportException err) { + close(); + throw err; + } catch (IOException | RuntimeException err) { + close(); + throw new TransportException(err.getMessage(), err); + } + } + + private void lsRefsImpl(Collection refSpecs, + String... additionalPatterns) throws IOException { + pckOut.writeString("command=" + COMMAND_LS_REFS); //$NON-NLS-1$ + // Add the user-agent + String agent = UserAgent.get(); + if (agent != null && isCapableOf(OPTION_AGENT)) { + pckOut.writeString(OPTION_AGENT + '=' + agent); + } + pckOut.writeDelim(); + pckOut.writeString("peel"); //$NON-NLS-1$ + pckOut.writeString("symrefs"); //$NON-NLS-1$ + for (String refPrefix : getRefPrefixes(refSpecs, additionalPatterns)) { + pckOut.writeString("ref-prefix " + refPrefix); //$NON-NLS-1$ + } + pckOut.end(); + final Map avail = new LinkedHashMap<>(); final Map symRefs = new LinkedHashMap<>(); - for (String option : capabilities) { - if (option.startsWith(CAPABILITY_SYMREF_PREFIX)) { - String[] symRef = option - .substring(CAPABILITY_SYMREF_PREFIX.length()) - .split(":", 2); //$NON-NLS-1$ - if (symRef.length == 2) { - symRefs.put(symRef[0], symRef[1]); - } + for (;;) { + String line = readLine(); + if (line == null) { + break; + } + // Expecting to get a line in the form "sha1 refname" + if (line.length() < 41 || line.charAt(40) != ' ') { + throw invalidRefAdvertisementLine(line); + } + String name = line.substring(41, line.length()); + final ObjectId id = toId(line, line.substring(0, 40)); + if (name.equals(".have")) { //$NON-NLS-1$ + additionalHaves.add(id); + } else { + processLineV2(line, id, name, avail, symRefs); } } - return symRefs; + updateWithSymRefs(avail, symRefs); + available(avail); + } + + private Collection getRefPrefixes(Collection refSpecs, + String... additionalPatterns) { + if (refSpecs.isEmpty() && (additionalPatterns == null + || additionalPatterns.length == 0)) { + return Collections.emptyList(); + } + Set patterns = new HashSet<>(); + if (additionalPatterns != null) { + Arrays.stream(additionalPatterns).filter(Objects::nonNull) + .forEach(patterns::add); + } + for (RefSpec spec : refSpecs) { + // TODO: for now we only do protocol V2 for fetch. For push + // RefSpecs, the logic would need to be different. (At the + // minimum, take spec.getDestination().) + String src = spec.getSource(); + if (ObjectId.isId(src)) { + continue; + } + if (spec.isWildcard()) { + patterns.add(src.substring(0, src.indexOf('*'))); + } else { + patterns.add(src); + patterns.add(Constants.R_REFS + src); + patterns.add(Constants.R_HEADS + src); + patterns.add(Constants.R_TAGS + src); + } + } + return patterns; + } + + private void readCapabilitiesV2() throws IOException { + // In git protocol V2, capabilities are different. If it's a key-value + // pair, the key may be a command name, and the value a space-separated + // list of capabilities for that command. We still store it in the same + // map as for protocol v0/v1. Protocol v2 code has to account for this. + for (;;) { + String line = readLine(); + if (line == null) { + break; + } + addCapability(line); + } + } + + private void addCapability(String capability) { + String parts[] = capability.split("=", 2); //$NON-NLS-1$ + if (parts.length == 2) { + remoteCapabilities.put(parts[0], parts[1]); + } + remoteCapabilities.put(capability, null); + } + + private ObjectId toId(String line, String value) + throws PackProtocolException { + try { + return ObjectId.fromString(value); + } catch (InvalidObjectIdException e) { + PackProtocolException ppe = invalidRefAdvertisementLine(line); + ppe.initCause(e); + throw ppe; + } + } + + private void processLineV1(String name, ObjectId id, Map avail) + throws IOException { + if (name.endsWith("^{}")) { //$NON-NLS-1$ + name = name.substring(0, name.length() - 3); + final Ref prior = avail.get(name); + if (prior == null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().advertisementCameBefore, name, name)); + } + if (prior.getPeeledObjectId() != null) { + throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$ + } + avail.put(name, new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, + prior.getObjectId(), id)); + } else { + final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag( + Ref.Storage.NETWORK, name, id)); + if (prior != null) { + throw duplicateAdvertisement(name); + } + } + } + + private void processLineV2(String line, ObjectId id, String rest, + Map avail, Map symRefs) + throws IOException { + String[] parts = rest.split(" "); //$NON-NLS-1$ + String name = parts[0]; + // Two attributes possible, symref-target or peeled + String symRefTarget = null; + String peeled = null; + for (int i = 1; i < parts.length; i++) { + if (parts[i].startsWith(REF_ATTR_SYMREF_TARGET)) { + if (symRefTarget != null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().duplicateRefAttribute, line)); + } + symRefTarget = parts[i] + .substring(REF_ATTR_SYMREF_TARGET.length()); + } else if (parts[i].startsWith(REF_ATTR_PEELED)) { + if (peeled != null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().duplicateRefAttribute, line)); + } + peeled = parts[i].substring(REF_ATTR_PEELED.length()); + } + if (peeled != null && symRefTarget != null) { + break; + } + } + Ref idRef; + if (peeled != null) { + idRef = new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, id, + toId(line, peeled)); + } else { + idRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, name, id); + } + Ref prior = avail.put(name, idRef); + if (prior != null) { + throw duplicateAdvertisement(name); + } + if (!StringUtils.isEmptyOrNull(symRefTarget)) { + symRefs.put(name, symRefTarget); + } } /** @@ -334,6 +543,22 @@ static void updateWithSymRefs(Map refMap, Map symRe } } } + // If HEAD is still in the symRefs map here, the real ref was not + // reported, but we know it must point to the object reported for HEAD. + // So fill it in in the refMap. + String headRefName = symRefs.get(Constants.HEAD); + if (headRefName != null && !refMap.containsKey(headRefName)) { + Ref headRef = refMap.get(Constants.HEAD); + if (headRef != null) { + ObjectId headObj = headRef.getObjectId(); + headRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, + headRefName, headObj); + refMap.put(headRefName, headRef); + headRef = new SymbolicRef(Constants.HEAD, headRef); + refMap.put(Constants.HEAD, headRef); + symRefs.remove(Constants.HEAD); + } + } } /** @@ -357,7 +582,7 @@ protected TransportException noRepository() { * @return whether this option is supported */ protected boolean isCapableOf(String option) { - return remoteCapablities.contains(option); + return remoteCapabilities.containsKey(option); } /** @@ -377,6 +602,17 @@ protected boolean wantCapability(StringBuilder b, String option) { return true; } + /** + * Return a capability value. + * + * @param option + * to get + * @return the value stored, if any. + */ + protected String getCapability(String option) { + return remoteCapabilities.get(option); + } + /** * Add user agent capability * @@ -385,7 +621,7 @@ protected boolean wantCapability(StringBuilder b, String option) { */ protected void addUserAgentCapability(StringBuilder b) { String a = UserAgent.get(); - if (a != null && UserAgent.hasAgent(remoteCapablities)) { + if (a != null && remoteCapabilities.get(OPTION_AGENT) != null) { b.append(' ').append(OPTION_AGENT).append('=').append(a); } } @@ -393,7 +629,8 @@ protected void addUserAgentCapability(StringBuilder b) { /** {@inheritDoc} */ @Override public String getPeerUserAgent() { - return UserAgent.getAgent(remoteCapablities, super.getPeerUserAgent()); + String agent = remoteCapabilities.get(OPTION_AGENT); + return agent != null ? agent : super.getPeerUserAgent(); } private PackProtocolException duplicateAdvertisement(String name) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index a2fb51f46..d344deac2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -16,18 +16,21 @@ import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; import org.eclipse.jgit.errors.PackProtocolException; +import org.eclipse.jgit.errors.RemoteRepositoryException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackLock; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -44,6 +47,7 @@ import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck; import org.eclipse.jgit.transport.PacketLineIn.AckNackResult; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.TemporaryBuffer; /** @@ -207,7 +211,10 @@ public abstract class BasePackFetchConnection extends BasePackConnection private int maxHaves; - /** RPC state, if {@link BasePackConnection#statelessRPC} is true. */ + /** + * RPC state, if {@link BasePackConnection#statelessRPC} is true or protocol + * V2 is used. + */ private TemporaryBuffer.Heap state; private PacketLineOut pckState; @@ -321,6 +328,13 @@ public Collection getPackLocks() { return Collections. emptyList(); } + private void clearState() { + walk.dispose(); + reachableCommits = null; + state = null; + pckState = null; + } + /** * Execute common ancestor negotiation and fetch the objects. * @@ -349,18 +363,34 @@ protected void doFetch(final ProgressMonitor monitor, markRefsAdvertised(); markReachable(have, maxTimeWanted(want)); + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + // Protocol V2 always is a "stateless" protocol, even over a + // bidirectional pipe: the server serves one "fetch" request and + // then forgets anything it has learned, so the next fetch + // request has to re-send all wants and previously determined + // common objects as "have"s again. + state = new TemporaryBuffer.Heap(Integer.MAX_VALUE); + pckState = new PacketLineOut(state); + try { + doFetchV2(monitor, want, outputStream); + } finally { + clearState(); + } + return; + } + // Protocol V0/1 if (statelessRPC) { state = new TemporaryBuffer.Heap(Integer.MAX_VALUE); pckState = new PacketLineOut(state); } - - if (sendWants(want)) { + PacketLineOut output = statelessRPC ? pckState : pckOut; + if (sendWants(want, output)) { + output.end(); + outNeedsEnd = false; negotiate(monitor); - walk.dispose(); - reachableCommits = null; - state = null; - pckState = null; + clearState(); receivePack(monitor, outputStream); } @@ -373,6 +403,185 @@ protected void doFetch(final ProgressMonitor monitor, } } + private void doFetchV2(ProgressMonitor monitor, Collection want, + OutputStream outputStream) throws IOException, CancelledException { + sideband = true; + negotiateBegin(); + + pckState.writeString("command=" + GitProtocolConstants.COMMAND_FETCH); //$NON-NLS-1$ + // Capabilities are sent as command arguments in protocol V2 + String agent = UserAgent.get(); + if (agent != null && isCapableOf(GitProtocolConstants.OPTION_AGENT)) { + pckState.writeString( + GitProtocolConstants.OPTION_AGENT + '=' + agent); + } + Set capabilities = new HashSet<>(); + String advertised = getCapability(GitProtocolConstants.COMMAND_FETCH); + if (!StringUtils.isEmptyOrNull(advertised)) { + capabilities.addAll(Arrays.asList(advertised.split("\\s+"))); //$NON-NLS-1$ + } + // Arguments + pckState.writeDelim(); + for (String capability : getCapabilitiesV2(capabilities)) { + pckState.writeString(capability); + } + if (!sendWants(want, pckState)) { + // We already have everything we wanted. + return; + } + // If we send something, we always close it properly ourselves. + outNeedsEnd = false; + + FetchStateV2 fetchState = new FetchStateV2(); + boolean sentDone = false; + for (;;) { + // The "state" buffer contains the full fetch request with all + // common objects found so far. + state.writeTo(out, monitor); + sentDone = sendNextHaveBatch(fetchState, pckOut, monitor); + if (sentDone) { + break; + } + if (readAcknowledgments(fetchState, pckIn, monitor)) { + // We got a "ready": next should be a patch file. + break; + } + // Note: C git reads and requires here (and after a packfile) a + // "0002" packet in stateless RPC transports (https). This "response + // end" packet is even mentioned in the protocol V2 technical + // documentation. However, it is not actually part of the public + // protocol; it occurs only in an internal protocol wrapper in the C + // git implementation. + } + clearState(); + String line = pckIn.readString(); + // If we sent a done, we may have an error reply here. + if (sentDone && line.startsWith("ERR ")) { //$NON-NLS-1$ + throw new RemoteRepositoryException(uri, line.substring(4)); + } + // "shallow-info", "wanted-refs", and "packfile-uris" would have to be + // handled here in that order. + if (!GitProtocolConstants.SECTION_PACKFILE.equals(line)) { + throw new PackProtocolException( + MessageFormat.format(JGitText.get().expectedGot, + GitProtocolConstants.SECTION_PACKFILE, line)); + } + receivePack(monitor, outputStream); + } + + /** + * Sends the next batch of "have"s and terminates the {@code output}. + * + * @param fetchState + * is updated with information about the number of items written, + * and whether to expect a packfile next + * @param output + * to write to + * @param monitor + * for progress reporting and cancellation + * @return {@code true} if a "done" was written and we should thus expect a + * packfile next + * @throws IOException + * on errors + * @throws CancelledException + * on cancellation + */ + private boolean sendNextHaveBatch(FetchStateV2 fetchState, + PacketLineOut output, ProgressMonitor monitor) + throws IOException, CancelledException { + long n = 0; + while (n < fetchState.havesToSend) { + final RevCommit c = walk.next(); + if (c == null) { + break; + } + output.writeString("have " + c.getId().name() + '\n'); //$NON-NLS-1$ + n++; + if (n % 10 == 0 && monitor.isCancelled()) { + throw new CancelledException(); + } + } + fetchState.havesTotal += n; + if (n == 0 + || (fetchState.hadAcks + && fetchState.havesWithoutAck > MAX_HAVES) + || fetchState.havesTotal > maxHaves) { + output.writeString("done\n"); //$NON-NLS-1$ + output.end(); + return true; + } + // Increment only after the test above. Of course we have no ACKs yet + // for the newly added "have"s, so it makes no sense to count them + // against the MAX_HAVES limit. + fetchState.havesWithoutAck += n; + output.end(); + fetchState.incHavesToSend(statelessRPC); + return false; + } + + /** + * Reads and processes acknowledgments, adding ACKed objects as "have"s to + * the global state {@link TemporaryBuffer}. + * + * @param fetchState + * to update + * @param input + * to read from + * @param monitor + * for progress reporting and cancellation + * @return {@code true} if a "ready" was received and a packfile is expected + * next + * @throws IOException + * on errors + * @throws CancelledException + * on cancellation + */ + private boolean readAcknowledgments(FetchStateV2 fetchState, + PacketLineIn input, ProgressMonitor monitor) + throws IOException, CancelledException { + String line = input.readString(); + if (!GitProtocolConstants.SECTION_ACKNOWLEDGMENTS.equals(line)) { + throw new PackProtocolException(MessageFormat.format( + JGitText.get().expectedGot, + GitProtocolConstants.SECTION_ACKNOWLEDGMENTS, line)); + } + MutableObjectId returnedId = new MutableObjectId(); + line = input.readString(); + boolean gotReady = false; + long n = 0; + while (!PacketLineIn.isEnd(line) && !PacketLineIn.isDelimiter(line)) { + AckNackResult ack = PacketLineIn.parseACKv2(line, returnedId); + // If we got a "ready", we just skip the remaining lines after + // having checked them for being valid. (Normally, the "ready" + // should be the last line anyway.) + if (!gotReady) { + if (ack == AckNackResult.ACK_COMMON) { + // markCommon appends the object to the "state" + markCommon(walk.parseAny(returnedId), ack, true); + fetchState.havesWithoutAck = 0; + fetchState.hadAcks = true; + } else if (ack == AckNackResult.ACK_READY) { + gotReady = true; + } + } + n++; + if (n % 10 == 0 && monitor.isCancelled()) { + throw new CancelledException(); + } + line = input.readString(); + } + if (gotReady) { + if (!PacketLineIn.isDelimiter(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0001", line)); //$NON-NLS-1$ + } + } else if (!PacketLineIn.isEnd(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0000", line)); //$NON-NLS-1$ + } + return gotReady; + } + /** {@inheritDoc} */ @Override public void close() { @@ -456,8 +665,8 @@ private void parseReachable(ObjectId id) { } } - private boolean sendWants(Collection want) throws IOException { - final PacketLineOut p = statelessRPC ? pckState : pckOut; + private boolean sendWants(Collection want, PacketLineOut p) + throws IOException { boolean first = true; for (Ref r : want) { ObjectId objectId = r.getObjectId(); @@ -479,10 +688,11 @@ private boolean sendWants(Collection want) throws IOException { final StringBuilder line = new StringBuilder(46); line.append("want "); //$NON-NLS-1$ line.append(objectId.name()); - if (first) { + if (first && TransferConfig.ProtocolVersion.V0 + .equals(getProtocolVersion())) { line.append(enableCapabilities()); - first = false; } + first = false; line.append('\n'); p.writeString(line.toString()); } @@ -492,11 +702,34 @@ private boolean sendWants(Collection want) throws IOException { if (!filterSpec.isNoOp()) { p.writeString(filterSpec.filterLine()); } - p.end(); - outNeedsEnd = false; return true; } + private Set getCapabilitiesV2(Set advertisedCapabilities) + throws TransportException { + Set capabilities = new LinkedHashSet<>(); + // Protocol V2 is implicitly capable of all these. + if (noProgress) { + capabilities.add(OPTION_NO_PROGRESS); + } + if (includeTags) { + capabilities.add(OPTION_INCLUDE_TAG); + } + if (allowOfsDelta) { + capabilities.add(OPTION_OFS_DELTA); + } + if (thinPack) { + capabilities.add(OPTION_THIN_PACK); + } + if (!filterSpec.isNoOp() + && !advertisedCapabilities.contains(OPTION_FILTER)) { + throw new PackProtocolException(uri, + JGitText.get().filterRequiresCapability); + } + // The FilterSpec will be added later in sendWants(). + return capabilities; + } + private String enableCapabilities() throws TransportException { final StringBuilder line = new StringBuilder(); if (noProgress) @@ -622,7 +855,7 @@ private void negotiate(ProgressMonitor monitor) throws IOException, // we need to continue to talk about other parts of // our local history. // - markCommon(walk.parseAny(ackId), anr); + markCommon(walk.parseAny(ackId), anr, statelessRPC); receivedAck = true; receivedContinue = true; havesSinceLastContinue = 0; @@ -757,16 +990,10 @@ private void markAdvertised(AnyObjectId id) { } } - private void markCommon(RevObject obj, AckNackResult anr) + private void markCommon(RevObject obj, AckNackResult anr, boolean useState) throws IOException { - if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) { - StringBuilder s; - - s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH); - s.append("have "); //$NON-NLS-1$ - s.append(obj.name()); - s.append('\n'); - pckState.writeString(s.toString()); + if (useState && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) { + pckState.writeString("have " + obj.name() + '\n'); //$NON-NLS-1$ obj.add(STATE); } obj.add(COMMON); @@ -806,4 +1033,31 @@ protected void onReceivePack() { private static class CancelledException extends Exception { private static final long serialVersionUID = 1L; } + + private static class FetchStateV2 { + + long havesToSend = 32; + + long havesTotal; + + // Set to true if we got at least one ACK in protocol V2. + boolean hadAcks; + + // Counts haves without ACK. Use as cutoff for negotiation only once + // hadAcks == true. + long havesWithoutAck; + + void incHavesToSend(boolean statelessRPC) { + if (statelessRPC) { + // Increase this quicker since connection setup costs accumulate + if (havesToSend < 16384) { + havesToSend *= 2; + } else { + havesToSend = havesToSend * 11 / 10; + } + } else { + havesToSend += 32; + } + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java index 0f1892a97..34bad6e02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -48,6 +48,7 @@ import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.util.StringUtils; class FetchProcess { /** Transport we will fetch over. */ @@ -79,7 +80,8 @@ class FetchProcess { toFetch = f; } - void execute(ProgressMonitor monitor, FetchResult result) + void execute(ProgressMonitor monitor, FetchResult result, + String initialBranch) throws NotSupportedException, TransportException { askFor.clear(); localUpdates.clear(); @@ -87,24 +89,64 @@ void execute(ProgressMonitor monitor, FetchResult result) packLocks.clear(); localRefs = null; + Throwable e1 = null; try { - executeImp(monitor, result); + executeImp(monitor, result, initialBranch); + } catch (NotSupportedException | TransportException err) { + e1 = err; + throw err; } finally { try { - for (PackLock lock : packLocks) - lock.unlock(); + for (PackLock lock : packLocks) { + lock.unlock(); + } } catch (IOException e) { + if (e1 != null) { + e.addSuppressed(e1); + } throw new TransportException(e.getMessage(), e); } } } + private boolean isInitialBranchMissing(Map refsMap, + String initialBranch) { + if (StringUtils.isEmptyOrNull(initialBranch) || refsMap.isEmpty()) { + return false; + } + if (refsMap.containsKey(initialBranch) + || refsMap.containsKey(Constants.R_HEADS + initialBranch) + || refsMap.containsKey(Constants.R_TAGS + initialBranch)) { + return false; + } + return true; + } + private void executeImp(final ProgressMonitor monitor, - final FetchResult result) throws NotSupportedException, - TransportException { - conn = transport.openFetch(); + final FetchResult result, String initialBranch) + throws NotSupportedException, TransportException { + final TagOpt tagopt = transport.getTagOpt(); + String getTags = (tagopt == TagOpt.NO_TAGS) ? null : Constants.R_TAGS; + String getHead = null; try { - result.setAdvertisedRefs(transport.getURI(), conn.getRefsMap()); + // If we don't have a HEAD yet, we're cloning and need to get the + // upstream HEAD, too. + Ref head = transport.local.exactRef(Constants.HEAD); + ObjectId id = head != null ? head.getObjectId() : null; + if (id == null || id.equals(ObjectId.zeroId())) { + getHead = Constants.HEAD; + } + } catch (IOException e) { + // Ignore + } + conn = transport.openFetch(toFetch, getTags, getHead); + try { + Map refsMap = conn.getRefsMap(); + if (isInitialBranchMissing(refsMap, initialBranch)) { + throw new TransportException(MessageFormat.format( + JGitText.get().remoteBranchNotFound, initialBranch)); + } + result.setAdvertisedRefs(transport.getURI(), refsMap); result.peerUserAgent = conn.getPeerUserAgent(); final Set matched = new HashSet<>(); for (RefSpec spec : toFetch) { @@ -119,7 +161,6 @@ private void executeImp(final ProgressMonitor monitor, } Collection additionalTags = Collections. emptyList(); - final TagOpt tagopt = transport.getTagOpt(); if (tagopt == TagOpt.AUTO_FOLLOW) additionalTags = expandAutoFollowTags(); else if (tagopt == TagOpt.FETCH_TAGS) @@ -253,7 +294,17 @@ private void reopenConnection() throws NotSupportedException, if (conn != null) return; - conn = transport.openFetch(); + // Build prefixes + Set prefixes = new HashSet<>(); + for (Ref toGet : askFor.values()) { + String src = toGet.getName(); + prefixes.add(src); + prefixes.add(Constants.R_REFS + src); + prefixes.add(Constants.R_HEADS + src); + prefixes.add(Constants.R_TAGS + src); + } + conn = transport.openFetch(Collections.emptyList(), + prefixes.toArray(new String[0])); // Since we opened a new connection we cannot be certain // that the system we connected to has the same exact set diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java index 35e2978bc..36fce7a3f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2013, Google Inc. + * Copyright (C) 2008, 2013 Google Inc. * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -247,6 +247,74 @@ public final class GitProtocolConstants { */ public static final String COMMAND_FETCH = "fetch"; //$NON-NLS-1$ + /** + * HTTP header to set by clients to request a specific git protocol version + * in the HTTP transport. + * + * @since 5.11 + */ + public static final String PROTOCOL_HEADER = "Git-Protocol"; //$NON-NLS-1$ + + /** + * Environment variable to set by clients to request a specific git protocol + * in the file:// and ssh:// transports. + * + * @since 5.11 + */ + public static final String PROTOCOL_ENVIRONMENT_VARIABLE = "GIT_PROTOCOL"; //$NON-NLS-1$ + + /** + * Protocol V2 ref advertisement attribute containing the peeled object id + * for annotated tags. + * + * @since 5.11 + */ + public static final String REF_ATTR_PEELED = "peeled:"; //$NON-NLS-1$ + + /** + * Protocol V2 ref advertisement attribute containing the name of the ref + * for symbolic refs. + * + * @since 5.11 + */ + public static final String REF_ATTR_SYMREF_TARGET = "symref-target:"; //$NON-NLS-1$ + + /** + * Protocol V2 acknowledgments section header. + * + * @since 5.11 + */ + public static final String SECTION_ACKNOWLEDGMENTS = "acknowledgments"; //$NON-NLS-1$ + + /** + * Protocol V2 packfile section header. + * + * @since 5.11 + */ + public static final String SECTION_PACKFILE = "packfile"; //$NON-NLS-1$ + + /** + * Protocol announcement for protocol version 1. This is the same as V0, + * except for this initial line. + * + * @since 5.11 + */ + public static final String VERSION_1 = "version 1"; //$NON-NLS-1$ + + /** + * Protocol announcement for protocol version 2. + * + * @since 5.11 + */ + public static final String VERSION_2 = "version 2"; //$NON-NLS-1$ + + /** + * Protocol request for protocol version 2. + * + * @since 5.11 + */ + public static final String VERSION_2_REQUEST = "version=2"; //$NON-NLS-1$ + enum MultiAck { OFF, CONTINUE, DETAILED; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java index 49c8b587d..febeb3ce9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java @@ -26,7 +26,7 @@ public abstract class HttpTransport extends Transport { * * @since 3.3 */ - protected static HttpConnectionFactory connectionFactory = new JDKHttpConnectionFactory(); + protected static volatile HttpConnectionFactory connectionFactory = new JDKHttpConnectionFactory(); /** * Get the {@link org.eclipse.jgit.transport.http.HttpConnectionFactory} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index 350311ecc..68c5b348a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -103,6 +103,38 @@ public PacketLineIn(InputStream in, long limit) { this.limit = limit; } + /** + * Parses a ACK/NAK line in protocol V2. + * + * @param line + * to parse + * @param returnedId + * in case of {@link AckNackResult#ACK_COMMON ACK_COMMON} + * @return one of {@link AckNackResult#NAK NAK}, + * {@link AckNackResult#ACK_COMMON ACK_COMMON}, or + * {@link AckNackResult#ACK_READY ACK_READY} + * @throws IOException + * on protocol or transport errors + */ + static AckNackResult parseACKv2(String line, MutableObjectId returnedId) + throws IOException { + if ("NAK".equals(line)) { //$NON-NLS-1$ + return AckNackResult.NAK; + } + if (line.startsWith("ACK ") && line.length() == 44) { //$NON-NLS-1$ + returnedId.fromString(line.substring(4, 44)); + return AckNackResult.ACK_COMMON; + } + if ("ready".equals(line)) { //$NON-NLS-1$ + return AckNackResult.ACK_READY; + } + if (line.startsWith("ERR ")) { //$NON-NLS-1$ + throw new PackProtocolException(line.substring(4)); + } + throw new PackProtocolException( + MessageFormat.format(JGitText.get().expectedACKNAKGot, line)); + } + AckNackResult readACK(MutableObjectId returnedId) throws IOException { final String line = readString(); if (line.length() == 0) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java index 6fc2042e1..77f0a7a51 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2010, Google Inc. - * Copyright (C) 2008-2009, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2010 Google Inc. + * Copyright (C) 2008, 2009 Robin Rosenberg + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -33,12 +33,15 @@ * against the underlying OutputStream. */ public class PacketLineOut { + private static final Logger log = LoggerFactory.getLogger(PacketLineOut.class); private final OutputStream out; private final byte[] lenbuffer; + private final boolean logEnabled; + private boolean flushOnEnd; private boolean usingSideband; @@ -50,9 +53,24 @@ public class PacketLineOut { * stream. */ public PacketLineOut(OutputStream outputStream) { + this(outputStream, true); + } + + /** + * Create a new packet line writer that potentially doesn't log. + * + * @param outputStream + * stream. + * @param enableLogging + * {@code false} to suppress all logging; {@code true} to log + * normally + * @since 5.11 + */ + public PacketLineOut(OutputStream outputStream, boolean enableLogging) { out = outputStream; lenbuffer = new byte[5]; flushOnEnd = true; + logEnabled = enableLogging; } /** @@ -139,9 +157,15 @@ public void writePacket(byte[] buf, int pos, int len) throws IOException { out.write(lenbuffer, 0, 4); } out.write(buf, pos, len); - if (log.isDebugEnabled()) { - String s = RawParseUtils.decode(UTF_8, buf, pos, len); - log.debug("git> " + s); //$NON-NLS-1$ + if (logEnabled && log.isDebugEnabled()) { + // Escape a trailing \n to avoid empty lines in the log. + if (len > 0 && buf[pos + len - 1] == '\n') { + log.debug( + "git> " + RawParseUtils.decode(UTF_8, buf, pos, len - 1) //$NON-NLS-1$ + + "\\n"); //$NON-NLS-1$ + } else { + log.debug("git> " + RawParseUtils.decode(UTF_8, buf, pos, len)); //$NON-NLS-1$ + } } } @@ -156,7 +180,9 @@ public void writePacket(byte[] buf, int pos, int len) throws IOException { public void writeDelim() throws IOException { formatLength(1); out.write(lenbuffer, 0, 4); - log.debug("git> 0001"); //$NON-NLS-1$ + if (logEnabled && log.isDebugEnabled()) { + log.debug("git> 0001"); //$NON-NLS-1$ + } } /** @@ -175,9 +201,12 @@ public void writeDelim() throws IOException { public void end() throws IOException { formatLength(0); out.write(lenbuffer, 0, 4); - log.debug("git> 0000"); //$NON-NLS-1$ - if (flushOnEnd) + if (logEnabled && log.isDebugEnabled()) { + log.debug("git> 0000"); //$NON-NLS-1$ + } + if (flushOnEnd) { flush(); + } } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java index 3adebba03..c525e6684 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2010, Google Inc. and others + * Copyright (C) 2008, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,6 +13,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SYMREF; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET; import java.io.IOException; import java.nio.ByteBuffer; @@ -287,7 +289,8 @@ public Set send(Collection refs) throws IOException { if (useProtocolV2) { String symrefPart = symrefs.containsKey(ref.getName()) - ? (" symref-target:" + symrefs.get(ref.getName())) //$NON-NLS-1$ + ? (' ' + REF_ATTR_SYMREF_TARGET + + symrefs.get(ref.getName())) : ""; //$NON-NLS-1$ String peelPart = ""; //$NON-NLS-1$ if (derefTags) { @@ -296,7 +299,8 @@ public Set send(Collection refs) throws IOException { } ObjectId peeledObjectId = ref.getPeeledObjectId(); if (peeledObjectId != null) { - peelPart = " peeled:" + peeledObjectId.getName(); //$NON-NLS-1$ + peelPart = ' ' + REF_ATTR_PEELED + + peeledObjectId.getName(); } } writeOne(objectId.getName() + " " + ref.getName() + symrefPart //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java new file mode 100644 index 000000000..23f670ae2 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2020, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.transport; + +import java.io.IOException; +import java.util.Map; + +/** + * A {@link RemoteSession} that supports passing environment variables to + * commands. + * + * @since 5.11 + */ +public interface RemoteSession2 extends RemoteSession { + + /** + * Creates a new remote {@link Process} to execute the given command. The + * returned process's streams exist and are connected, and execution of the + * process is already started. + * + * @param commandName + * command to execute + * @param environment + * environment variables to pass on + * @param timeout + * timeout value, in seconds, for creating the remote process + * @return a new remote process, already started + * @throws java.io.IOException + * may be thrown in several cases. For example, on problems + * opening input or output streams or on problems connecting or + * communicating with the remote host. For the latter two cases, + * a TransportException may be thrown (a subclass of + * java.io.IOException). + */ + Process exec(String commandName, Map environment, + int timeout) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java index cc577fa11..83ffd4123 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2009, Google Inc. and others + * Copyright (C) 2008, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -21,6 +21,7 @@ import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.storage.file.LazyObjectIdSetFile; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Config.SectionParser; import org.eclipse.jgit.lib.ObjectChecker; import org.eclipse.jgit.lib.ObjectIdSet; @@ -60,11 +61,19 @@ public enum FsckMode { } /** - * A git configuration variable for which versions of the Git protocol to prefer. - * Used in protocol.version. + * A git configuration variable for which versions of the Git protocol to + * prefer. Used in protocol.version. + * + * @since 5.9 */ - enum ProtocolVersion { + public enum ProtocolVersion { + /** + * Git wire protocol version 0 (the default). + */ V0("0"), //$NON-NLS-1$ + /** + * Git wire protocol version 2. + */ V2("2"); //$NON-NLS-1$ final String name; @@ -73,6 +82,15 @@ enum ProtocolVersion { this.name = name; } + /** + * Returns version number + * + * @return string version + */ + public String version() { + return name; + } + @Nullable static ProtocolVersion parse(@Nullable String name) { if (name == null) { @@ -83,6 +101,9 @@ static ProtocolVersion parse(@Nullable String name) { return v; } } + if ("1".equals(name)) { //$NON-NLS-1$ + return V0; + } return null; } } @@ -177,7 +198,9 @@ public TransferConfig(Config rc) { "uploadpack", "allowreachablesha1inwant", false); allowFilter = rc.getBoolean( "uploadpack", "allowfilter", false); - protocolVersion = ProtocolVersion.parse(rc.getString("protocol", null, "version")); + protocolVersion = ProtocolVersion.parse(rc + .getString(ConfigConstants.CONFIG_PROTOCOL_SECTION, null, + ConfigConstants.CONFIG_KEY_VERSION)); hideRefs = rc.getStringList("uploadpack", null, "hiderefs"); allowSidebandAll = rc.getBoolean( "uploadpack", "allowsidebandall", false); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index 2ddd0a612..5b781ac25 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2009, Google Inc. + * Copyright (C) 2008, 2009 Google Inc. * Copyright (C) 2008, Marek Zawirski * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -39,6 +39,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.AbortedByHookException; import org.eclipse.jgit.errors.NotSupportedException; import org.eclipse.jgit.errors.TransportException; @@ -774,6 +775,10 @@ private static String findTrackingRefName(final String remoteName, private PrintStream hookOutRedirect; private PrePushHook prePush; + + @Nullable + TransferConfig.ProtocolVersion protocol; + /** * Create a new transport instance. * @@ -789,6 +794,7 @@ protected Transport(Repository local, URIish uri) { final TransferConfig tc = local.getConfig().get(TransferConfig.KEY); this.local = local; this.uri = uri; + this.protocol = tc.protocolVersion; this.objectChecker = tc.newObjectChecker(); this.credentialsProvider = CredentialsProvider.getDefault(); prePush = Hooks.prePush(local, hookOutRedirect); @@ -1225,9 +1231,52 @@ public void setPushOptions(List pushOptions) { * the remote connection could not be established or object * copying (if necessary) failed or update specification was * incorrect. + * @since 5.11 */ public FetchResult fetch(final ProgressMonitor monitor, - Collection toFetch) throws NotSupportedException, + Collection toFetch) + throws NotSupportedException, TransportException { + return fetch(monitor, toFetch, null); + } + + /** + * Fetch objects and refs from the remote repository to the local one. + *

    + * This is a utility function providing standard fetch behavior. Local + * tracking refs associated with the remote repository are automatically + * updated if this transport was created from a + * {@link org.eclipse.jgit.transport.RemoteConfig} with fetch RefSpecs + * defined. + * + * @param monitor + * progress monitor to inform the user about our processing + * activity. Must not be null. Use + * {@link org.eclipse.jgit.lib.NullProgressMonitor} if progress + * updates are not interesting or necessary. + * @param toFetch + * specification of refs to fetch locally. May be null or the + * empty collection to use the specifications from the + * RemoteConfig. Source for each RefSpec can't be null. + * @param branch + * the initial branch to check out when cloning the repository. + * Can be specified as ref name (refs/heads/master), + * branch name (master) or tag name + * (v1.2.3). The default is to use the branch + * pointed to by the cloned repository's HEAD and can be + * requested by passing {@code null} or HEAD. + * @return information describing the tracking refs updated. + * @throws org.eclipse.jgit.errors.NotSupportedException + * this transport implementation does not support fetching + * objects. + * @throws org.eclipse.jgit.errors.TransportException + * the remote connection could not be established or object + * copying (if necessary) failed or update specification was + * incorrect. + * @since 5.11 + */ + public FetchResult fetch(final ProgressMonitor monitor, + Collection toFetch, String branch) + throws NotSupportedException, TransportException { if (toFetch == null || toFetch.isEmpty()) { // If the caller did not ask for anything use the defaults. @@ -1257,7 +1306,7 @@ public FetchResult fetch(final ProgressMonitor monitor, } final FetchResult result = new FetchResult(); - new FetchProcess(this, toFetch).execute(monitor, result); + new FetchProcess(this, toFetch).execute(monitor, result, branch); local.autoGC(monitor); @@ -1452,6 +1501,43 @@ public Collection findRemoteRefUpdatesFor( public abstract FetchConnection openFetch() throws NotSupportedException, TransportException; + /** + * Begins a new connection for fetching from the remote repository. + *

    + * If the transport has no local repository, the fetch connection can only + * be used for reading remote refs. + *

    + *

    + * If the server supports git protocol V2, the {@link RefSpec}s and the + * additional patterns, if any, are used to restrict the server's ref + * advertisement to matching refs only. + *

    + *

    + * Transports that want to support git protocol V2 must override + * this; the default implementation ignores its arguments and calls + * {@link #openFetch()}. + *

    + * + * @param refSpecs + * that will be fetched via + * {@link FetchConnection#fetch(ProgressMonitor, Collection, java.util.Set, OutputStream)} later + * @param additionalPatterns + * that will be set as ref prefixes if the server supports git + * protocol V2; {@code null} values are ignored + * + * @return a fresh connection to fetch from the remote repository. + * @throws org.eclipse.jgit.errors.NotSupportedException + * the implementation does not support fetching. + * @throws org.eclipse.jgit.errors.TransportException + * the remote connection could not be established. + * @since 5.11 + */ + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return openFetch(); + } + /** * Begins a new connection for pushing into the remote repository. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java index 820ec1a67..a1914b618 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2008, Marek Zawirski * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -22,6 +22,7 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.UnknownHostException; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.Set; @@ -94,6 +95,13 @@ public FetchConnection openFetch() throws TransportException { return new TcpFetchConnection(); } + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return new TcpFetchConnection(refSpecs, additionalPatterns); + } + /** {@inheritDoc} */ @Override public PushConnection openPush() throws TransportException { @@ -113,7 +121,6 @@ Socket openConnection() throws TransportException { final Socket s = new Socket(); try { final InetAddress host = InetAddress.getByName(uri.getHost()); - s.bind(null); s.connect(new InetSocketAddress(host, port), tms); } catch (IOException c) { try { @@ -130,7 +137,8 @@ Socket openConnection() throws TransportException { return s; } - void service(String name, PacketLineOut pckOut) + void service(String name, PacketLineOut pckOut, + TransferConfig.ProtocolVersion gitProtocol) throws IOException { final StringBuilder cmd = new StringBuilder(); cmd.append(name); @@ -144,6 +152,11 @@ void service(String name, PacketLineOut pckOut) cmd.append(uri.getPort()); } cmd.append('\0'); + if (TransferConfig.ProtocolVersion.V2.equals(gitProtocol)) { + cmd.append('\0'); + cmd.append(GitProtocolConstants.VERSION_2_REQUEST); + cmd.append('\0'); + } pckOut.writeString(cmd.toString()); pckOut.flush(); } @@ -152,6 +165,11 @@ class TcpFetchConnection extends BasePackFetchConnection { private Socket sock; TcpFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + TcpFetchConnection(Collection refSpecs, + String... additionalPatterns) throws TransportException { super(TransportGitAnon.this); sock = openConnection(); try { @@ -162,13 +180,19 @@ class TcpFetchConnection extends BasePackFetchConnection { sOut = new BufferedOutputStream(sOut); init(sIn, sOut); - service("git-upload-pack", pckOut); //$NON-NLS-1$ + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + service("git-upload-pack", pckOut, gitProtocol); //$NON-NLS-1$ } catch (IOException err) { close(); throw new TransportException(uri, JGitText.get().remoteHungUpUnexpectedly, err); } - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } @Override @@ -201,7 +225,7 @@ class TcpPushConnection extends BasePackPushConnection { sOut = new BufferedOutputStream(sOut); init(sIn, sOut); - service("git-receive-pack", pckOut); //$NON-NLS-1$ + service("git-receive-pack", pckOut, null); //$NON-NLS-1$ } catch (IOException err) { close(); throw new TransportException(uri, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java index b9cb2484d..19ed4fbcc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Marek Zawirski * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -19,11 +19,13 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import org.eclipse.jgit.errors.NoRemoteRepositoryException; @@ -144,6 +146,13 @@ public FetchConnection openFetch() throws TransportException { return new SshFetchConnection(); } + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return new SshFetchConnection(refSpecs, additionalPatterns); + } + /** {@inheritDoc} */ @Override public PushConnection openPush() throws TransportException { @@ -196,29 +205,38 @@ private static boolean useExtSession() { return SystemReader.getInstance().getenv("GIT_SSH") != null; //$NON-NLS-1$ } - private class ExtSession implements RemoteSession { + private class ExtSession implements RemoteSession2 { + @Override public Process exec(String command, int timeout) throws TransportException { + return exec(command, null, timeout); + } + + @Override + public Process exec(String command, Map environment, + int timeout) throws TransportException { String ssh = SystemReader.getInstance().getenv("GIT_SSH"); //$NON-NLS-1$ boolean putty = ssh.toLowerCase(Locale.ROOT).contains("plink"); //$NON-NLS-1$ List args = new ArrayList<>(); args.add(ssh); - if (putty - && !ssh.toLowerCase(Locale.ROOT).contains("tortoiseplink")) //$NON-NLS-1$ + if (putty && !ssh.toLowerCase(Locale.ROOT) + .contains("tortoiseplink")) {//$NON-NLS-1$ args.add("-batch"); //$NON-NLS-1$ + } if (0 < getURI().getPort()) { args.add(putty ? "-P" : "-p"); //$NON-NLS-1$ //$NON-NLS-2$ args.add(String.valueOf(getURI().getPort())); } - if (getURI().getUser() != null) + if (getURI().getUser() != null) { args.add(getURI().getUser() + "@" + getURI().getHost()); //$NON-NLS-1$ - else + } else { args.add(getURI().getHost()); + } args.add(command); - ProcessBuilder pb = createProcess(args); + ProcessBuilder pb = createProcess(args, environment); try { return pb.start(); } catch (IOException err) { @@ -226,9 +244,13 @@ public Process exec(String command, int timeout) } } - private ProcessBuilder createProcess(List args) { + private ProcessBuilder createProcess(List args, + Map environment) { ProcessBuilder pb = new ProcessBuilder(); pb.command(args); + if (environment != null) { + pb.environment().putAll(environment); + } File directory = local != null ? local.getDirectory() : null; if (directory != null) { pb.environment().put(Constants.GIT_DIR_KEY, @@ -249,10 +271,31 @@ class SshFetchConnection extends BasePackFetchConnection { private StreamCopyThread errorThread; SshFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + SshFetchConnection(Collection refSpecs, + String... additionalPatterns) throws TransportException { super(TransportGitSsh.this); try { - process = getSession().exec(commandFor(getOptionUploadPack()), - getTimeout()); + RemoteSession session = getSession(); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + if (session instanceof RemoteSession2 + && TransferConfig.ProtocolVersion.V2 + .equals(gitProtocol)) { + process = ((RemoteSession2) session).exec( + commandFor(getOptionUploadPack()), Collections + .singletonMap( + GitProtocolConstants.PROTOCOL_ENVIRONMENT_VARIABLE, + GitProtocolConstants.VERSION_2_REQUEST), + getTimeout()); + } else { + process = session.exec(commandFor(getOptionUploadPack()), + getTimeout()); + } final MessageWriter msg = new MessageWriter(); setMessageWriter(msg); @@ -272,7 +315,9 @@ class SshFetchConnection extends BasePackFetchConnection { } try { - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } catch (NoRemoteRepositoryException notFound) { final String msgs = getMessages(); checkExecFailure(process.exitValue(), getOptionUploadPack(), diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 6768387e6..2e5d18dc1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Shawn O. Pearce * Copyright (C) 2013, Matthias Sohn - * Copyright (C) 2017, Thomas Wolf and others + * Copyright (C) 2017, 2020 Thomas Wolf and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -33,14 +33,15 @@ import static org.eclipse.jgit.util.HttpSupport.METHOD_GET; import static org.eclipse.jgit.util.HttpSupport.METHOD_POST; +import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.net.HttpCookie; import java.net.MalformedURLException; import java.net.Proxy; @@ -49,10 +50,12 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.GeneralSecurityException; import java.security.cert.CertPathBuilderException; import java.security.cert.CertPathValidatorException; import java.security.cert.CertificateException; @@ -75,6 +78,7 @@ import javax.net.ssl.SSLHandshakeException; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.NotSupportedException; @@ -95,6 +99,8 @@ import org.eclipse.jgit.transport.HttpAuthMethod.Type; import org.eclipse.jgit.transport.HttpConfig.HttpRedirectMode; import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.transport.http.HttpConnectionFactory; +import org.eclipse.jgit.transport.http.HttpConnectionFactory2; import org.eclipse.jgit.util.HttpSupport; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; @@ -132,6 +138,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$ + private static final byte[] VERSION = "version" //$NON-NLS-1$ + .getBytes(StandardCharsets.US_ASCII); + /** * Accept-Encoding header in the HTTP request * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html). @@ -257,6 +266,12 @@ public Transport open(URIish uri, Repository local, String remoteName) private boolean sslFailure = false; + private HttpConnectionFactory factory; + + private HttpConnectionFactory2.GitSession gitSession; + + private boolean factoryUsed; + /** * All stored cookies bound to this repo (independent of the baseUrl) */ @@ -279,6 +294,7 @@ public Transport open(URIish uri, Repository local, String remoteName) sslVerify = http.isSslVerify(); cookieFile = getCookieFileFromConfig(http); relevantCookies = filterCookies(cookieFile, baseUrl); + factory = HttpTransport.getConnectionFactory(); } private URL toURL(URIish urish) throws MalformedURLException { @@ -321,6 +337,7 @@ protected void setURI(URIish uri) throws NotSupportedException { sslVerify = http.isSslVerify(); cookieFile = getCookieFileFromConfig(http); relevantCookies = filterCookies(cookieFile, baseUrl); + factory = HttpTransport.getConnectionFactory(); } /** @@ -339,11 +356,15 @@ public void setUseSmartHttp(boolean on) { @SuppressWarnings("resource") // Closed by caller private FetchConnection getConnection(HttpConnection c, InputStream in, - String service) throws IOException { + String service, Collection refSpecs, + String... additionalPatterns) throws IOException { BaseConnection f; if (isSmartHttp(c, service)) { - readSmartHeaders(in, service); - f = new SmartHttpFetchConnection(in); + InputStream withMark = in.markSupported() ? in + : new BufferedInputStream(in); + readSmartHeaders(withMark, service); + f = new SmartHttpFetchConnection(withMark, refSpecs, + additionalPatterns); } else { // Assume this server doesn't support smart HTTP fetch // and fall back on dumb object walking. @@ -353,15 +374,98 @@ private FetchConnection getConnection(HttpConnection c, InputStream in, return (FetchConnection) f; } + /** + * Sets the {@link HttpConnectionFactory} to be used by this + * {@link TransportHttp} instance. + *

    + * If no factory is set explicitly, the {@link TransportHttp} instance uses + * the {@link HttpTransport#getConnectionFactory() globally defined + * factory}. + *

    + * + * @param customFactory + * the {@link HttpConnectionFactory} to use + * @throws IllegalStateException + * if an HTTP/HTTPS connection has already been opened on this + * {@link TransportHttp} instance + * @since 5.11 + */ + public void setHttpConnectionFactory( + @NonNull HttpConnectionFactory customFactory) { + if (factoryUsed) { + throw new IllegalStateException(JGitText.get().httpFactoryInUse); + } + factory = customFactory; + } + + /** + * Retrieves the {@link HttpConnectionFactory} used by this + * {@link TransportHttp} instance. + * + * @return the {@link HttpConnectionFactory} + * @since 5.11 + */ + @NonNull + public HttpConnectionFactory getHttpConnectionFactory() { + return factory; + } + + /** + * Sets preemptive Basic HTTP authentication. If the given {@code username} + * or {@code password} is empty or {@code null}, no preemptive + * authentication will be done. If {@code username} and {@code password} are + * set, they will override authority information from the URI + * ("user:password@"). + *

    + * If the connection encounters redirects, the pre-authentication will be + * cleared if the redirect goes to a different host. + *

    + * + * @param username + * to use + * @param password + * to use + * @throws IllegalStateException + * if an HTTP/HTTPS connection has already been opened on this + * {@link TransportHttp} instance + * @since 5.11 + */ + public void setPreemptiveBasicAuthentication(String username, + String password) { + if (factoryUsed) { + throw new IllegalStateException(JGitText.get().httpPreAuthTooLate); + } + if (StringUtils.isEmptyOrNull(username) + || StringUtils.isEmptyOrNull(password)) { + authMethod = authFromUri(currentUri); + } else { + HttpAuthMethod basic = HttpAuthMethod.Type.BASIC.method(null); + basic.authorize(username, password); + authMethod = basic; + } + } + /** {@inheritDoc} */ @Override public FetchConnection openFetch() throws TransportException, NotSupportedException { + return openFetch(Collections.emptyList()); + } + + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { final String service = SVC_UPLOAD_PACK; try { - final HttpConnection c = connect(service); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + HttpConnection c = connect(service, gitProtocol); try (InputStream in = openInputStream(c)) { - return getConnection(c, in, service); + return getConnection(c, in, service, refSpecs, + additionalPatterns); } } catch (NotSupportedException | TransportException err) { throw err; @@ -456,8 +560,9 @@ public PushConnection openPush() throws NotSupportedException, private PushConnection smartPush(String service, HttpConnection c, InputStream in) throws IOException, TransportException { - readSmartHeaders(in, service); - SmartHttpPushConnection p = new SmartHttpPushConnection(in); + BufferedInputStream inBuf = new BufferedInputStream(in); + readSmartHeaders(inBuf, service); + SmartHttpPushConnection p = new SmartHttpPushConnection(inBuf); p.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER)); return p; } @@ -465,7 +570,10 @@ private PushConnection smartPush(String service, HttpConnection c, /** {@inheritDoc} */ @Override public void close() { - // No explicit connections are maintained. + if (gitSession != null) { + gitSession.close(); + gitSession = null; + } } /** @@ -492,9 +600,40 @@ private NoRemoteRepositoryException createNotFoundException(URIish u, return new NoRemoteRepositoryException(u, text); } + private HttpAuthMethod authFromUri(URIish u) { + String user = u.getUser(); + String pass = u.getPass(); + if (user != null && pass != null) { + try { + // User/password are _not_ application/x-www-form-urlencoded. In + // particular the "+" sign would be replaced by a space. + user = URLDecoder.decode(user.replace("+", "%2B"), //$NON-NLS-1$ //$NON-NLS-2$ + StandardCharsets.UTF_8.name()); + pass = URLDecoder.decode(pass.replace("+", "%2B"), //$NON-NLS-1$ //$NON-NLS-2$ + StandardCharsets.UTF_8.name()); + HttpAuthMethod basic = HttpAuthMethod.Type.BASIC.method(null); + basic.authorize(user, pass); + return basic; + } catch (IllegalArgumentException + | UnsupportedEncodingException e) { + LOG.warn(JGitText.get().httpUserInfoDecodeError, u); + } + } + return HttpAuthMethod.Type.NONE.method(null); + } + private HttpConnection connect(String service) throws TransportException, NotSupportedException { + return connect(service, null); + } + + private HttpConnection connect(String service, + TransferConfig.ProtocolVersion protocolVersion) + throws TransportException, NotSupportedException { URL u = getServiceURL(service); + if (HttpAuthMethod.Type.NONE.equals(authMethod.getType())) { + authMethod = authFromUri(currentUri); + } int authAttempts = 1; int redirects = 0; Collection ignoreTypes = null; @@ -507,6 +646,11 @@ private HttpConnection connect(String service) } else { conn.setRequestProperty(HDR_ACCEPT, "*/*"); //$NON-NLS-1$ } + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + conn.setRequestProperty( + GitProtocolConstants.PROTOCOL_HEADER, + GitProtocolConstants.VERSION_2_REQUEST); + } final int status = HttpSupport.response(conn); processResponseCookies(conn); switch (status) { @@ -796,7 +940,13 @@ private URIish redirect(URL currentUrl, String location, String checkFor, } try { URI redirectTo = new URI(location); + // Reset authentication if the redirect has user/password info or + // if the host is different. + boolean resetAuth = !StringUtils + .isEmptyOrNull(redirectTo.getUserInfo()); + String currentHost = currentUrl.getHost(); redirectTo = currentUrl.toURI().resolve(redirectTo); + resetAuth = resetAuth || !currentHost.equals(redirectTo.getHost()); String redirected = redirectTo.toASCIIString(); if (!isValidRedirect(baseUrl, redirected, checkFor)) { throw new TransportException(uri, @@ -805,6 +955,9 @@ private URIish redirect(URL currentUrl, String location, String checkFor, } redirected = redirected.substring(0, redirected.indexOf(checkFor)); URIish result = new URIish(redirected); + if (resetAuth) { + authMethod = HttpAuthMethod.Type.NONE.method(null); + } if (LOG.isInfoEnabled()) { LOG.info(MessageFormat.format(JGitText.get().redirectHttp, uri.setPass(null), @@ -885,9 +1038,20 @@ protected HttpConnection httpOpen(String method, URL u, } final Proxy proxy = HttpSupport.proxyFor(proxySelector, u); - HttpConnection conn = connectionFactory.create(u, proxy); + factoryUsed = true; + HttpConnection conn = factory.create(u, proxy); - if (!sslVerify && "https".equals(u.getProtocol())) { //$NON-NLS-1$ + if (gitSession == null && (factory instanceof HttpConnectionFactory2)) { + gitSession = ((HttpConnectionFactory2) factory).newSession(); + } + if (gitSession != null) { + try { + gitSession.configure(conn, sslVerify); + } catch (GeneralSecurityException e) { + throw new IOException(e.getMessage(), e); + } + } else if (!sslVerify && "https".equals(u.getProtocol())) { //$NON-NLS-1$ + // Backwards compatibility HttpSupport.disableSslVerify(conn); } @@ -1148,20 +1312,37 @@ private boolean isGzipContent(HttpConnection c) { private void readSmartHeaders(InputStream in, String service) throws IOException { - // A smart reply will have a '#' after the first 4 bytes, but - // a dumb reply cannot contain a '#' until after byte 41. Do a + // A smart protocol V0 reply will have a '#' after the first 4 bytes, + // but a dumb reply cannot contain a '#' until after byte 41. Do a // quick check to make sure its a smart reply before we parse // as a pkt-line stream. // - final byte[] magic = new byte[5]; + // There appears to be a confusion about this in protocol V2. Github + // sends the # service line as a git (not http) header also when + // protocol V2 is used. Gitlab also does so. JGit's UploadPack doesn't, + // and thus Gerrit also does not. + final byte[] magic = new byte[14]; + if (!in.markSupported()) { + throw new TransportException(uri, + JGitText.get().inputStreamMustSupportMark); + } + in.mark(14); IO.readFully(in, magic, 0, magic.length); + // Did we get 000dversion 2 or similar? (Canonical is 000eversion 2\n, + // but JGit and thus Gerrit omits the \n.) + if (Arrays.equals(Arrays.copyOfRange(magic, 4, 11), VERSION) + && magic[12] >= '1' && magic[12] <= '9') { + // It's a smart server doing version 1 or greater, but not sending + // the # service line header. Don't consume the version line. + in.reset(); + return; + } if (magic[4] != '#') { throw new TransportException(uri, MessageFormat.format( JGitText.get().expectedPktLineWithService, RawParseUtils.decode(magic))); } - - final PacketLineIn pckIn = new PacketLineIn(new UnionInputStream( - new ByteArrayInputStream(magic), in)); + in.reset(); + final PacketLineIn pckIn = new PacketLineIn(in); final String exp = "# service=" + service; //$NON-NLS-1$ final String act = pckIn.readString(); if (!exp.equals(act)) { @@ -1327,12 +1508,24 @@ class SmartHttpFetchConnection extends BasePackFetchConnection { SmartHttpFetchConnection(InputStream advertisement) throws TransportException { + this(advertisement, Collections.emptyList()); + } + + SmartHttpFetchConnection(InputStream advertisement, + Collection refSpecs, String... additionalPatterns) + throws TransportException { super(TransportHttp.this); statelessRPC = true; init(advertisement, DisabledOutputStream.INSTANCE); outNeedsEnd = false; - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + // Must be protocol V2 + LongPollService service = new LongPollService(SVC_UPLOAD_PACK, + getProtocolVersion()); + init(service.getInputStream(), service.getOutputStream()); + lsRefs(refSpecs, additionalPatterns); + } } @Override @@ -1340,7 +1533,8 @@ protected void doFetch(final ProgressMonitor monitor, final Collection want, final Set have, final OutputStream outputStream) throws TransportException { try { - svc = new MultiRequestService(SVC_UPLOAD_PACK); + svc = new MultiRequestService(SVC_UPLOAD_PACK, + getProtocolVersion()); init(svc.getInputStream(), svc.getOutputStream()); super.doFetch(monitor, want, have, outputStream); } finally { @@ -1369,7 +1563,8 @@ class SmartHttpPushConnection extends BasePackPushConnection { protected void doPush(final ProgressMonitor monitor, final Map refUpdates, OutputStream outputStream) throws TransportException { - final Service svc = new MultiRequestService(SVC_RECEIVE_PACK); + final Service svc = new MultiRequestService(SVC_RECEIVE_PACK, + getProtocolVersion()); init(svc.getInputStream(), svc.getOutputStream()); super.doPush(monitor, refUpdates, outputStream); } @@ -1389,10 +1584,14 @@ abstract class Service { protected final HttpExecuteStream execute; + protected final TransferConfig.ProtocolVersion protocolVersion; + final UnionInputStream in; - Service(String serviceName) { + Service(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { this.serviceName = serviceName; + this.protocolVersion = protocolVersion; this.requestType = "application/x-" + serviceName + "-request"; //$NON-NLS-1$ //$NON-NLS-2$ this.responseType = "application/x-" + serviceName + "-result"; //$NON-NLS-1$ //$NON-NLS-2$ @@ -1408,6 +1607,10 @@ void openStream() throws IOException { conn.setDoOutput(true); conn.setRequestProperty(HDR_CONTENT_TYPE, requestType); conn.setRequestProperty(HDR_ACCEPT, responseType); + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + conn.setRequestProperty(GitProtocolConstants.PROTOCOL_HEADER, + GitProtocolConstants.VERSION_2_REQUEST); + } } void sendRequest() throws IOException { @@ -1663,8 +1866,9 @@ protected OutputStream overflow() throws IOException { class MultiRequestService extends Service { boolean finalRequest; - MultiRequestService(String serviceName) { - super(serviceName); + MultiRequestService(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { + super(serviceName, protocolVersion); } /** Keep opening send-receive pairs to the given URI. */ @@ -1701,11 +1905,10 @@ void execute() throws IOException { /** Service for maintaining a single long-poll connection. */ class LongPollService extends Service { - /** - * @param serviceName - */ - LongPollService(String serviceName) { - super(serviceName); + + LongPollService(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { + super(serviceName, protocolVersion); } /** Only open one send-receive request. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java index 403f98d86..77d1419ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java @@ -1,9 +1,9 @@ /* * Copyright (C) 2007, Dave Watson - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Marek Zawirski * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2020 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -153,11 +154,17 @@ private Repository openRepo() throws TransportException { /** {@inheritDoc} */ @Override public FetchConnection openFetch() throws TransportException { + return openFetch(Collections.emptyList()); + } + + @Override + public FetchConnection openFetch(Collection refSpecs, + String... additionalPatterns) throws TransportException { final String up = getOptionUploadPack(); if (!"git-upload-pack".equals(up) //$NON-NLS-1$ - && !"git upload-pack".equals(up)) //$NON-NLS-1$ - return new ForkLocalFetchConnection(); - + && !"git upload-pack".equals(up)) {//$NON-NLS-1$ + return new ForkLocalFetchConnection(refSpecs, additionalPatterns); + } UploadPackFactory upf = (Void req, Repository db) -> createUploadPack(db); return new InternalFetchConnection<>(this, upf, null, openRepo()); @@ -193,6 +200,23 @@ public void close() { */ protected Process spawn(String cmd) throws TransportException { + return spawn(cmd, null); + } + + /** + * Spawn process + * + * @param cmd + * command + * @param protocolVersion + * to use + * @return a {@link java.lang.Process} object. + * @throws org.eclipse.jgit.errors.TransportException + * if any. + */ + private Process spawn(String cmd, + TransferConfig.ProtocolVersion protocolVersion) + throws TransportException { try { String[] args = { "." }; //$NON-NLS-1$ ProcessBuilder proc = local.getFS().runInShell(cmd, args); @@ -208,7 +232,10 @@ protected Process spawn(String cmd) env.remove("GIT_GRAFT_FILE"); //$NON-NLS-1$ env.remove("GIT_INDEX_FILE"); //$NON-NLS-1$ env.remove("GIT_NO_REPLACE_OBJECTS"); //$NON-NLS-1$ - + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + env.put(GitProtocolConstants.PROTOCOL_ENVIRONMENT_VARIABLE, + GitProtocolConstants.VERSION_2_REQUEST); + } return proc.start(); } catch (IOException err) { throw new TransportException(uri, err.getMessage(), err); @@ -221,12 +248,21 @@ class ForkLocalFetchConnection extends BasePackFetchConnection { private Thread errorReaderThread; ForkLocalFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + ForkLocalFetchConnection(Collection refSpecs, + String... additionalPatterns) throws TransportException { super(TransportLocal.this); final MessageWriter msg = new MessageWriter(); setMessageWriter(msg); - uploadPack = spawn(getOptionUploadPack()); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + uploadPack = spawn(getOptionUploadPack(), gitProtocol); final InputStream upErr = uploadPack.getErrorStream(); errorReaderThread = new StreamCopyThread(upErr, msg.getRawStream()); @@ -239,7 +275,9 @@ class ForkLocalFetchConnection extends BasePackFetchConnection { upOut = new BufferedOutputStream(upOut); init(upIn, upOut); - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 1242ef1b4..7f1ddaab2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2010, Google Inc. and others + * Copyright (C) 2008, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -33,6 +33,7 @@ import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND_64K; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_THIN_PACK; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2_REQUEST; import static org.eclipse.jgit.util.RefMap.toRefMap; import java.io.ByteArrayOutputStream; @@ -709,7 +710,7 @@ public boolean isSideBand() throws RequestNotYetReadException { * @since 5.0 */ public void setExtraParameters(Collection params) { - this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ + this.clientRequestedV2 = params.contains(VERSION_2_REQUEST); } /** @@ -722,7 +723,8 @@ public void setCachedPackUriProvider(@Nullable CachedPackUriProvider p) { } private boolean useProtocolV2() { - return ProtocolVersion.V2.equals(transferConfig.protocolVersion) + return (transferConfig.protocolVersion == null + || ProtocolVersion.V2.equals(transferConfig.protocolVersion)) && clientRequestedV2; } @@ -1191,17 +1193,18 @@ private void fetchV2(PacketLineOut pckOut) throws IOException { if (req.wasDoneReceived()) { processHaveLines(req.getPeerHas(), ObjectId.zeroId(), - new PacketLineOut(NullOutputStream.INSTANCE), + new PacketLineOut(NullOutputStream.INSTANCE, false), accumulator); } else { - pckOut.writeString("acknowledgments\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_ACKNOWLEDGMENTS + '\n'); for (ObjectId id : req.getPeerHas()) { if (walk.getObjectReader().has(id)) { pckOut.writeString("ACK " + id.getName() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ } } processHaveLines(req.getPeerHas(), ObjectId.zeroId(), - new PacketLineOut(NullOutputStream.INSTANCE), + new PacketLineOut(NullOutputStream.INSTANCE, false), accumulator); if (okToGiveUp()) { pckOut.writeString("ready\n"); //$NON-NLS-1$ @@ -1243,7 +1246,8 @@ private void fetchV2(PacketLineOut pckOut) throws IOException { if (!pckOut.isUsingSideband()) { // sendPack will write "packfile\n" for us if sideband-all is used. // But sideband-all is not used, so we have to write it ourselves. - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_PACKFILE + '\n'); } accumulator.timeNegotiating = Duration @@ -1955,8 +1959,8 @@ private static void checkNotAdvertisedWants(UploadPack up, .map(objId -> objectIdToRevObject(objWalk, objId)) .filter(Objects::nonNull); // Ignore missing tips - ObjectReachabilityChecker reachabilityChecker = objWalk - .createObjectReachabilityChecker(); + ObjectReachabilityChecker reachabilityChecker = reader + .createObjectReachabilityChecker(objWalk); Optional unreachable = reachabilityChecker .areAllReachable(wantsAsObjs, startersAsObjs); if (unreachable.isPresent()) { @@ -1967,8 +1971,8 @@ private static void checkNotAdvertisedWants(UploadPack up, } // All wants are commits, we can use ReachabilityChecker - ReachabilityChecker reachabilityChecker = walk - .createReachabilityChecker(); + ReachabilityChecker reachabilityChecker = reader + .createReachabilityChecker(walk); Stream reachableCommits = importantRefsFirst(visibleRefs) .map(UploadPack::refToObjectId) @@ -2327,7 +2331,8 @@ else if (ref.getName().startsWith(Constants.R_HEADS)) // for us if provided a PackfileUriConfig. In this case, we // are not providing a PackfileUriConfig, so we have to // write this line ourselves. - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_PACKFILE + '\n'); } } pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java index c44838ab8..a6b20451d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java @@ -510,6 +510,7 @@ private boolean downloadPackedObject(final ProgressMonitor monitor, // and attach it to the local repository so we can use // all of the contained objects. // + Throwable e1 = null; try { pack.downloadPack(monitor); } catch (IOException err) { @@ -519,6 +520,7 @@ private boolean downloadPackedObject(final ProgressMonitor monitor, // an alternate. // recordError(id, err); + e1 = err; continue; } finally { // If the pack was good its in the local repository @@ -531,6 +533,9 @@ private boolean downloadPackedObject(final ProgressMonitor monitor, if (pack.tmpIdx != null) FileUtils.delete(pack.tmpIdx); } catch (IOException e) { + if (e1 != null) { + e.addSuppressed(e1); + } throw new TransportException(e.getMessage(), e); } packItr.remove(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnectionFactory2.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnectionFactory2.java new file mode 100644 index 000000000..88abc6016 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnectionFactory2.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2020 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport.http; + +import java.io.IOException; +import java.security.GeneralSecurityException; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * A {@link HttpConnectionFactory} that supports client-side sessions that can + * maintain state and configure connections. + * + * @since 5.11 + */ +public interface HttpConnectionFactory2 extends HttpConnectionFactory { + + /** + * Creates a new {@link GitSession} instance that can be used with + * connections created by this {@link HttpConnectionFactory} instance. + * + * @return a new {@link GitSession} + */ + @NonNull + GitSession newSession(); + + /** + * A {@code GitSession} groups the multiple HTTP connections + * {@link org.eclipse.jgit.transport.TransportHttp TransportHttp} uses for + * the requests it makes during a git fetch or push. A {@code GitSession} + * can maintain client-side HTTPS state and can configure individual + * connections. + */ + interface GitSession { + + /** + * Configure a just created {@link HttpConnection}. + * + * @param connection + * to configure; created by the same + * {@link HttpConnectionFactory} instance + * @param sslVerify + * whether SSL is to be verified + * @return the configured {@connection} + * @throws IOException + * if the connection cannot be configured + * @throws GeneralSecurityException + * if the connection cannot be configured + */ + @NonNull + HttpConnection configure(@NonNull HttpConnection connection, + boolean sslVerify) throws IOException, GeneralSecurityException; + + /** + * Closes the {@link GitSession}, releasing any internal state. + */ + void close(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java index b39f1579b..1b5d1b3c4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Christian Halstrick and others + * Copyright (C) 2013, 2020 Christian Halstrick and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,6 +12,18 @@ import java.io.IOException; import java.net.Proxy; import java.net.URL; +import java.security.GeneralSecurityException; +import java.text.MessageFormat; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.transport.http.DelegatingSSLSocketFactory; +import org.eclipse.jgit.util.HttpSupport; /** * A factory returning instances of @@ -19,17 +31,70 @@ * * @since 3.3 */ -public class JDKHttpConnectionFactory implements HttpConnectionFactory { - /** {@inheritDoc} */ +public class JDKHttpConnectionFactory implements HttpConnectionFactory2 { + @Override public HttpConnection create(URL url) throws IOException { return new JDKHttpConnection(url); } - /** {@inheritDoc} */ @Override public HttpConnection create(URL url, Proxy proxy) throws IOException { return new JDKHttpConnection(url, proxy); } + + @Override + public GitSession newSession() { + return new JdkConnectionSession(); + } + + private static class JdkConnectionSession implements GitSession { + + private SSLContext securityContext; + + private SSLSocketFactory socketFactory; + + @Override + public JDKHttpConnection configure(HttpConnection connection, + boolean sslVerify) throws GeneralSecurityException { + if (!(connection instanceof JDKHttpConnection)) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().httpWrongConnectionType, + JDKHttpConnection.class.getName(), + connection.getClass().getName())); + } + JDKHttpConnection conn = (JDKHttpConnection) connection; + String scheme = conn.getURL().getProtocol(); + if (!"https".equals(scheme) || sslVerify) { //$NON-NLS-1$ + // sslVerify == true: use the JDK defaults + return conn; + } + if (securityContext == null) { + securityContext = SSLContext.getInstance("TLS"); //$NON-NLS-1$ + TrustManager[] trustAllCerts = { + new NoCheckX509TrustManager() }; + securityContext.init(null, trustAllCerts, null); + socketFactory = new DelegatingSSLSocketFactory( + securityContext.getSocketFactory()) { + + @Override + protected void configure(SSLSocket socket) { + HttpSupport.configureTLS(socket); + } + }; + } + conn.setHostnameVerifier((name, session) -> true); + ((HttpsURLConnection) conn.wrappedUrlConnection) + .setSSLSocketFactory(socketFactory); + return conn; + } + + @Override + public void close() { + securityContext = null; + socketFactory = null; + } + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/NoCheckX509TrustManager.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/NoCheckX509TrustManager.java new file mode 100644 index 000000000..5cd4fb4c8 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/NoCheckX509TrustManager.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2016, 2020 JGit contributors + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + * + * Contributors: + * Saša Živkov 2016 - initial API + * Thomas Wolf 2020 - extracted from HttpSupport + */ +package org.eclipse.jgit.transport.http; + +import java.security.cert.X509Certificate; + +import javax.net.ssl.X509TrustManager; + +/** + * A {@link X509TrustManager} that doesn't verify anything. Can be used for + * skipping SSL checks. + * + * @since 5.11 + */ +public class NoCheckX509TrustManager implements X509TrustManager { + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + + @Override + public void checkClientTrusted(X509Certificate[] certs, + String authType) { + // no check + } + + @Override + public void checkServerTrusted(X509Certificate[] certs, + String authType) { + // no check + } +} \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 72278dc9c..55b7d6279 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -2,7 +2,7 @@ * Copyright (C) 2008, Shawn O. Pearce * Copyright (C) 2010, Christian Halstrick * Copyright (C) 2010, Matthias Sohn - * Copyright (C) 2012-2020, Robin Rosenberg and others + * Copyright (C) 2012-2021, Robin Rosenberg and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -63,6 +63,7 @@ import org.eclipse.jgit.treewalk.TreeWalk.OperationType; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.ExecutionResult; +import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Holder; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.Paths; @@ -799,7 +800,10 @@ protected void init(Entry[] list) { if (Constants.DOT_GIT.equals(name)) continue; if (Constants.DOT_GIT_IGNORE.equals(name)) - ignoreNode = new PerDirectoryIgnoreNode(e); + ignoreNode = new PerDirectoryIgnoreNode( + TreeWalk.pathOf(path, 0, pathOffset) + + Constants.DOT_GIT_IGNORE, + e); if (Constants.DOT_GIT_ATTRIBUTES.equals(name)) attributesNode = new PerDirectoryAttributesNode(e); if (i != o) @@ -983,8 +987,9 @@ public boolean isModified(DirCacheEntry entry, boolean forceContentCheck, return true; } else if (ObjectId.zeroId().compareTo(idBuffer, idOffset) == 0) { - return new File(repository.getWorkTree(), - entry.getPathString()).list().length > 0; + Path p = repository.getWorkTree().toPath() + .resolve(entry.getPathString()); + return FileUtils.hasFiles(p); } return false; } else if (mode == FileMode.SYMLINK.getBits()) @@ -1272,17 +1277,20 @@ public String toString() { /** Magic type indicating we know rules exist, but they aren't loaded. */ private static class PerDirectoryIgnoreNode extends IgnoreNode { - final Entry entry; + protected final Entry entry; - PerDirectoryIgnoreNode(Entry entry) { + private final String name; + + PerDirectoryIgnoreNode(String name, Entry entry) { super(Collections. emptyList()); + this.name = name; this.entry = entry; } IgnoreNode load() throws IOException { IgnoreNode r = new IgnoreNode(); try (InputStream in = entry.openInputStream()) { - r.parse(in); + r.parse(name, in); } return r.getRules().isEmpty() ? null : r; } @@ -1293,7 +1301,7 @@ private static class RootIgnoreNode extends PerDirectoryIgnoreNode { final Repository repository; RootIgnoreNode(Entry entry, Repository repository) { - super(entry); + super(entry != null ? entry.getName() : null, entry); this.repository = repository; } @@ -1327,7 +1335,7 @@ private static void loadRulesFromFile(IgnoreNode r, File exclude) throws FileNotFoundException, IOException { if (FS.DETECTED.exists(exclude)) { try (FileInputStream in = new FileInputStream(exclude)) { - r.parse(in); + r.parse(exclude.getAbsolutePath(), in); } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index d8cab358e..0946f645f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -22,7 +22,6 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; -import java.io.PrintStream; import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; @@ -337,6 +336,9 @@ public static FileStoreAttributes get(Path path) { try { path = path.toAbsolutePath(); Path dir = Files.isDirectory(path) ? path : path.getParent(); + if (dir == null) { + return FALLBACK_FILESTORE_ATTRIBUTES; + } FileStoreAttributes cached = attrCacheByPath.get(dir); if (cached != null) { return cached; @@ -511,7 +513,10 @@ private static Duration measureMinimalRacyInterval(Path dir) { } private static void write(Path p, String body) throws IOException { - FileUtils.mkdirs(p.getParent().toFile(), true); + Path parent = p.getParent(); + if (parent != null) { + FileUtils.mkdirs(parent.toFile(), true); + } try (Writer w = new OutputStreamWriter(Files.newOutputStream(p), UTF_8)) { w.write(body); @@ -1867,18 +1872,18 @@ public ProcessResult runHookIfPresent(Repository repository, * @throws org.eclipse.jgit.api.errors.JGitInternalException * if we fail to run the hook somehow. Causes may include an * interrupted process or I/O errors. - * @since 4.0 + * @since 5.11 */ public ProcessResult runHookIfPresent(Repository repository, final String hookName, - String[] args, PrintStream outRedirect, PrintStream errRedirect, + String[] args, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws JGitInternalException { return new ProcessResult(Status.NOT_SUPPORTED); } /** * See - * {@link #runHookIfPresent(Repository, String, String[], PrintStream, PrintStream, String)} + * {@link #runHookIfPresent(Repository, String, String[], OutputStream, OutputStream, String)} * . Should only be called by FS supporting shell scripts execution. * * @param repository @@ -1903,11 +1908,11 @@ public ProcessResult runHookIfPresent(Repository repository, * @throws org.eclipse.jgit.api.errors.JGitInternalException * if we fail to run the hook somehow. Causes may include an * interrupted process or I/O errors. - * @since 4.0 + * @since 5.11 */ protected ProcessResult internalRunHookIfPresent(Repository repository, - final String hookName, String[] args, PrintStream outRedirect, - PrintStream errRedirect, String stdinArgs) + final String hookName, String[] args, OutputStream outRedirect, + OutputStream errRedirect, String stdinArgs) throws JGitInternalException { File hookFile = findHook(repository, hookName); if (hookFile == null || hookName == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java index fb63dc02b..946d81c73 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java @@ -16,7 +16,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; -import java.io.PrintStream; +import java.io.OutputStream; import java.nio.charset.Charset; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileStore; @@ -268,7 +268,7 @@ String shellQuote(String cmd) { /** {@inheritDoc} */ @Override public ProcessResult runHookIfPresent(Repository repository, String hookName, - String[] args, PrintStream outRedirect, PrintStream errRedirect, + String[] args, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws JGitInternalException { return internalRunHookIfPresent(repository, hookName, args, outRedirect, errRedirect, stdinArgs); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java index d53bff78e..add549817 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java @@ -13,7 +13,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; -import java.io.PrintStream; +import java.io.OutputStream; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; @@ -139,7 +139,7 @@ public String relativize(String base, String other) { /** {@inheritDoc} */ @Override public ProcessResult runHookIfPresent(Repository repository, String hookName, - String[] args, PrintStream outRedirect, PrintStream errRedirect, + String[] args, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws JGitInternalException { return internalRunHookIfPresent(repository, hookName, args, outRedirect, errRedirect, stdinArgs); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index aa39a4464..b9dd9baa6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -43,6 +43,7 @@ import java.util.Locale; import java.util.Random; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; @@ -801,6 +802,23 @@ static boolean isFile(File file) { return Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS); } + /** + * Whether the path is a directory with files in it. + * + * @param dir + * directory path + * @return {@code true} if the given directory path contains files + * @throws IOException + * on any I/O errors accessing the path + * + * @since 5.11 + */ + public static boolean hasFiles(Path dir) throws IOException { + try (Stream stream = Files.list(dir)) { + return stream.findAny().isPresent(); + } + } + /** * Whether the given file can be executed. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java index 04b3eab50..23a73faf8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java @@ -24,21 +24,18 @@ import java.net.URLEncoder; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; import java.text.MessageFormat; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; import java.util.Set; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocket; import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.transport.http.NoCheckX509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -301,40 +298,13 @@ public static Proxy proxyFor(ProxySelector proxySelector, URL u) */ public static void disableSslVerify(HttpConnection conn) throws IOException { - final TrustManager[] trustAllCerts = new TrustManager[] { - new DummyX509TrustManager() }; + TrustManager[] trustAllCerts = { + new NoCheckX509TrustManager() }; try { conn.configure(null, trustAllCerts, null); - conn.setHostnameVerifier(new DummyHostnameVerifier()); + conn.setHostnameVerifier((name, session) -> true); } catch (KeyManagementException | NoSuchAlgorithmException e) { - throw new IOException(e.getMessage()); - } - } - - private static class DummyX509TrustManager implements X509TrustManager { - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - } - - @Override - public void checkClientTrusted(X509Certificate[] certs, - String authType) { - // no check - } - - @Override - public void checkServerTrusted(X509Certificate[] certs, - String authType) { - // no check - } - } - - private static class DummyHostnameVerifier implements HostnameVerifier { - @Override - public boolean verify(String hostname, SSLSession session) { - // always accept - return true; + throw new IOException(e.getMessage(), e); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java index 680d90392..6d5694e43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java @@ -246,7 +246,7 @@ public static int read(ReadableByteChannel channel, byte[] dst, int off, * buffer that must be fully populated, [off, off+len). * @param off * position within the buffer to start writing to. - * @return number of bytes in buffer or stream, whichever is shortest + * @return number of bytes read * @throws java.io.IOException * there was an error reading from the stream. */ @@ -254,8 +254,8 @@ public static int readFully(InputStream fd, byte[] dst, int off) throws IOException { int r; int len = 0; - while ((r = fd.read(dst, off, dst.length - off)) >= 0 - && len < dst.length) { + while (off < dst.length + && (r = fd.read(dst, off, dst.length - off)) >= 0) { off += r; len += r; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java new file mode 100644 index 000000000..cf06172c1 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2021, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import java.text.MessageFormat; +import java.util.Locale; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.GpgSignatureVerifier.TrustLevel; +import org.eclipse.jgit.lib.PersonIdent; + +/** + * Utilities for signature verification. + * + * @since 5.11 + */ +public final class SignatureUtils { + + private SignatureUtils() { + // No instantiation + } + + /** + * Writes information about a signature verification to a string. + * + * @param verification + * to show + * @param creator + * of the object verified; used for time zone information + * @param formatter + * to use for dates + * @return a textual representation of the {@link SignatureVerification}, + * using LF as line separator + */ + public static String toString(SignatureVerification verification, + PersonIdent creator, GitDateFormatter formatter) { + StringBuilder result = new StringBuilder(); + // Use the creator's timezone for the signature date + PersonIdent dateId = new PersonIdent(creator, + verification.getCreationDate()); + result.append(MessageFormat.format(JGitText.get().verifySignatureMade, + formatter.formatDate(dateId))); + result.append('\n'); + result.append(MessageFormat.format( + JGitText.get().verifySignatureKey, + verification.getKeyFingerprint().toUpperCase(Locale.ROOT))); + result.append('\n'); + if (!StringUtils.isEmptyOrNull(verification.getSigner())) { + result.append( + MessageFormat.format(JGitText.get().verifySignatureIssuer, + verification.getSigner())); + result.append('\n'); + } + String msg; + if (verification.getVerified()) { + if (verification.isExpired()) { + msg = JGitText.get().verifySignatureExpired; + } else { + msg = JGitText.get().verifySignatureGood; + } + } else { + msg = JGitText.get().verifySignatureBad; + } + result.append(MessageFormat.format(msg, verification.getKeyUser())); + if (!TrustLevel.UNKNOWN.equals(verification.getTrustLevel())) { + result.append(' ' + MessageFormat + .format(JGitText.get().verifySignatureTrust, verification + .getTrustLevel().name().toLowerCase(Locale.ROOT))); + } + result.append('\n'); + msg = verification.getMessage(); + if (!StringUtils.isEmptyOrNull(msg)) { + result.append(msg); + result.append('\n'); + } + return result.toString(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 447f417e7..54fd539f6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -56,9 +56,9 @@ public abstract class SystemReader { private static final SystemReader DEFAULT; - private static Boolean isMacOS; + private static volatile Boolean isMacOS; - private static Boolean isWindows; + private static volatile Boolean isWindows; static { SystemReader r = new Default(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java index 1f0fedda6..562eb05dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java @@ -230,11 +230,16 @@ public byte[] toByteArray(int limit) throws IOException { if (Integer.MAX_VALUE < len) throw new OutOfMemoryError( JGitText.get().lengthExceedsMaximumArraySize); - final byte[] out = new byte[(int) len]; + int length = (int) len; + final byte[] out = new byte[length]; int outPtr = 0; for (Block b : blocks) { - System.arraycopy(b.buffer, 0, out, outPtr, b.count); - outPtr += b.count; + int toCopy = Math.min(length - outPtr, b.count); + System.arraycopy(b.buffer, 0, out, outPtr, toCopy); + outPtr += toCopy; + if (outPtr == length) { + break; + } } return out; } @@ -460,6 +465,30 @@ public byte[] toByteArray() throws IOException { return out; } + @Override + public byte[] toByteArray(int limit) throws IOException { + if (onDiskFile == null) { + return super.toByteArray(limit); + } + final long len = Math.min(length(), limit); + if (Integer.MAX_VALUE < len) { + throw new OutOfMemoryError( + JGitText.get().lengthExceedsMaximumArraySize); + } + final byte[] out = new byte[(int) len]; + try (FileInputStream in = new FileInputStream(onDiskFile)) { + int read = 0; + int chunk; + while ((chunk = in.read(out, read, out.length - read)) >= 0) { + read += chunk; + if (read == out.length) { + break; + } + } + } + return out; + } + @Override public void writeTo(OutputStream os, ProgressMonitor pm) throws IOException { diff --git a/pom.xml b/pom.xml index cf581feed..bd362965c 100644 --- a/pom.xml +++ b/pom.xml @@ -151,8 +151,8 @@ 1.8 ${project.build.directory}/META-INF/MANIFEST.MF - 5.9.0.202009080501-r - 2.4.0 + 5.10.0.202012080955-r + 2.6.0 0.1.55 1.1.1 1.1.7 @@ -162,20 +162,20 @@ 1.19 4.3.1 3.1.0 - 9.4.30.v20200611 - 0.14.3 - 4.5.10 - 4.4.12 + 9.4.36.v20210114 + 0.14.4 + 4.5.13 + 4.4.14 1.7.30 1.2.15 3.2.0 1.7.0 - 2.8.2 + 2.8.6 1.65 - 4.1.3 + 4.2.0 3.1.1 3.0.0 - 3.0.0-M4 + 3.0.0-M5 ${maven-surefire-plugin-version} 3.8.1 2.8.8 @@ -299,7 +299,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.13.0 + 3.14.0 utf-8 100 @@ -347,7 +347,7 @@ org.apache.maven.wagon wagon-ssh - 3.4.0 + 3.4.2 @@ -389,7 +389,7 @@ org.springframework.boot spring-boot-maven-plugin - 2.1.5.RELEASE + 2.4.1 @@ -408,7 +408,7 @@ - 3.6.2 + 3.6.3 @@ -896,7 +896,7 @@ org.eclipse.jdt ecj - 3.23.0 + 3.24.0