From 5a232c9509cc4035dad8490b0d9cd790b49c75ae Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 15 Jun 2021 13:44:58 +0200 Subject: [PATCH 01/14] Add Eclipse code of conduct and security policy This was requested in the 5.12 release review. Bug: https://gitlab.eclipse.org/eclipsefdn/emo-team/emo/-/issues/58 Change-Id: Ifbfa4c239e8ce0da6d5571362a12c20c399b64e4 --- CODE_OF_CONDUCT.md | 46 +++++++++++++++++++++++++++++++++++++++++++ SECURITY.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 SECURITY.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..14db7e6fa --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Community Code of Conduct + +**Version 1.2 +August 19, 2020** + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as community members, contributors, committers, and project leaders pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +With the support of the Eclipse Foundation staff (the “Staff”), project committers and leaders are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project committers and leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the Eclipse Foundation project or its community in public spaces. Examples of representing a project or community include posting via an official social media account, or acting as a project representative at an online or offline event. Representation of a project may be further defined and clarified by project committers, leaders, or the EMO. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the Staff at codeofconduct@eclipse.org. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The Staff is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project committers or leaders who do not follow the Code of Conduct in good faith may face temporary or permanent repercussions as determined by the Staff. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org) , version 1.4, available at [https://www.contributor-covenant.org/version/1/4/code-of-conduct.html](https://www.contributor-covenant.org/version/1/4/code-of-conduct/) \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..e6f57c698 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,49 @@ + +_ISO 27005 defines vulnerability as: + "A weakness of an asset or group of assets that can be exploited by one or more threats."_ + +## The Eclipse Security Team + +The Eclipse Security Team provides help and advice to Eclipse projects +on vulnerability issues and is the first point of contact +for handling security vulnerabilities. +Members of the Security Team are committers on Eclipse Projects +and members of the Eclipse Architecture Council. + +Contact the [Eclipse Security Team](mailto:security@eclipse.org). + +**Note that, as a matter of policy, the security team does not open attachments.** + +## Reporting a Security Vulnerability + +Vulnerabilities can be reported either via email to the Eclipse Security Team +or directly with a project via the Eclipse Foundation's Bugzilla instance. + +The general security mailing list address is security@eclipse.org. +Members of the Eclipse Security Team will receive messages sent to this address. +This address should be used only for reporting undisclosed vulnerabilities; +regular issue reports and questions unrelated to vulnerabilities in Eclipse software +will be ignored. +Note that this email address is not encrypted. + +The community is also encouraged to report vulnerabilities using the +[Eclipse Foundation's Bugzilla instance](https://bugs.eclipse.org/bugs/enter_bug.cgi?product=Community&component=Vulnerability%20Reports&keywords=security&groups=Security_Advisories). +Note that you will require an Eclipse Foundation account to create an issue report, +but by doing so you will be able to participate directly in the resolution of the issue. + +Issue reports related to vulnerabilities must be marked as "committers-only", +either automatically by clicking the provided link, by the reporter, +or by a committer during the triage process. +Note that issues marked "committers-only" are visible to all Eclipse committers. +By default, a "committers-only" issue is also accessible to the reporter +and individuals explicitly indicated in the "cc" list. + +## Disclosure + +Disclosure is initially limited to the reporter and all Eclipse Committers, +but is expanded to include other individuals, and the general public. +The timing and manner of disclosure is governed by the +[Eclipse Security Policy](https://www.eclipse.org/security/policy.php). + +Publicly disclosed issues are listed on the +[Disclosed Vulnerabilities Page](https://www.eclipse.org/security/known.php). \ No newline at end of file From 45b16ebcc152944d7474ed9d6a36a70b4d9e582f Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 15 Jun 2021 13:59:23 +0200 Subject: [PATCH 02/14] CONTRIBUTING: Use standard markdown format Change-Id: Ic64a62ca29950fb6a4c3c8a31bee8173c005ea4e --- CONTRIBUTING.md | 44 ++++++++++++++++---------------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4818589ff..2fc355001 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,44 +1,32 @@ -Contributing -============ +# Contributing -Please refer to the contributor guide for all the details: +Please refer to the [contributor guide](https://wiki.eclipse.org/EGit/Contributor_Guide) for all the details. -https://wiki.eclipse.org/EGit/Contributor_Guide - - -Reporting bugs --------------- +## Reporting bugs For anything other than small changes, it's a good idea to open a bug report for it (in case one doesn't already exist). This gives others the -chance to give input and is useful for tracking. Create one here: +chance to give input and is useful for tracking. +[Create JGit bugs here](https://bugs.eclipse.org/bugs/enter_bug.cgi?product=JGit). -https://bugs.eclipse.org/bugs/enter_bug.cgi?product=JGit +## Submitting changes - -Submitting changes ------------------- - -We use Gerrit to review all changes by committers or contributors before -they are merged: - -https://git.eclipse.org/r/ - -Make sure you have an account and have set up the `commit-msg` hook +- We use [Gerrit](https://git.eclipse.org/r/) to review all changes by committers +or contributors before they are merged. +- Make sure you have an account and have set up the `commit-msg` hook before committing. - -When committing your changes, see the contributor guide or other commits +- When committing your changes, see the contributor guide or other commits on what your commit message should include. - -Run the following to push your change for review (with `username` +- Run the following to push your change for review (with `username` replaced by your Gerrit username): - git push ssh://username@git.eclipse.org:29418/jgit/jgit.git HEAD:refs/for/master +```bash +git push ssh://username@git.eclipse.org:29418/jgit/jgit.git HEAD:refs/for/master +``` -Add the link to the review as a comment on the bug report, so that +- Add the link to the review as a comment on the bug report, so that people coming from the bug report can find it. - -Then wait for someone to review your change. If there is something to be +- Then wait for someone to review your change. If there is something to be corrected, amend your commit and push it again. Have fun :). From ee01bbca22f8c75355af081046fa5923c88bc488 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 15 Jun 2021 14:01:00 +0200 Subject: [PATCH 03/14] CONTRIBUTING: add explicit link to ECA This was requested in the 5.12 release review. Bug: https://gitlab.eclipse.org/eclipsefdn/emo-team/emo/-/issues/58 Change-Id: I21abe960412a7bc7a0d2f3bdfc4a1d031a700c3b --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2fc355001..e0859d2c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,7 @@ # Contributing Please refer to the [contributor guide](https://wiki.eclipse.org/EGit/Contributor_Guide) for all the details. +Contributions require that you sign the [Eclipse Contributor Agreement](https://www.eclipse.org/legal/ECA.php). ## Reporting bugs From 79d86c4a57da75750f32648b37ff8caca645edba Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Wed, 16 Jun 2021 18:14:17 +0200 Subject: [PATCH 04/14] [releng] japicmp: update last release version The baseline for the 5.13 release is 5.12.0.202106070339-r. Change-Id: Ic4730ca296117d7a7f6e86228ce2393a748c3c06 Signed-off-by: Thomas Wolf --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2bf2de150..b2e79058c 100644 --- a/pom.xml +++ b/pom.xml @@ -151,7 +151,7 @@ 1.8 ${project.build.directory}/META-INF/MANIFEST.MF - 5.11.0.202103091610-r + 5.12.0.202106070339-r 2.6.0 0.1.55 1.1.1 From ce61341bd12c2554d76499a894ecfc854754edfb Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 19 Jun 2021 15:41:37 +0200 Subject: [PATCH 05/14] [sshd] Log the full KEX negotiation result Apache MINA sshd doesn't log all values, but it'd be very helpful to debug KEX problems. Change-Id: I5a19023c929f39caccde489d51a6fdca711fe5ff Signed-off-by: Thomas Wolf --- .../transport/sshd/JGitClientSession.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java index 8183a92b9..f9e80121e 100644 --- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java +++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java @@ -39,6 +39,7 @@ import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.common.io.IoSession; import org.apache.sshd.common.io.IoWriteFuture; +import org.apache.sshd.common.kex.KexProposalOption; import org.apache.sshd.common.util.Readable; import org.apache.sshd.common.util.buffer.Buffer; import org.eclipse.jgit.errors.InvalidPatternException; @@ -200,6 +201,24 @@ public void messageReceived(Readable buffer) throws Exception { } } + @Override + protected Map setNegotiationResult( + Map guess) { + Map result = super.setNegotiationResult( + guess); + // This should be doable with a SessionListener, too, but I don't see + // how to add a listener in time to catch the negotiation end for sure + // given that the super-constructor already starts KEX. + // + // TODO: This override can be removed once we use sshd 2.8.0. + if (log.isDebugEnabled()) { + result.forEach((option, value) -> log.debug( + "setNegotiationResult({}) Kex: {} = {}", this, //$NON-NLS-1$ + option.getDescription(), value)); + } + return result; + } + @Override protected String resolveAvailableSignaturesProposal( FactoryManager manager) { From 2fc600b7ccd1128f65d564cf1445605eaa223873 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 16 Jun 2021 17:43:25 +0200 Subject: [PATCH 06/14] Update jetty to 9.4.42.v20210604 Change-Id: Ie0abae5686064ccfeb34e8af0d98471162036b00 --- WORKSPACE | 30 +++++++------- .../org.eclipse.jgit.target/jgit-4.10.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.11.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.12.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.13.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.14.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.15.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.16.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.17.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.18.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.19.target | 40 +++++++++---------- .../jgit-4.20-staging.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.6.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.7.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.8.target | 40 +++++++++---------- .../org.eclipse.jgit.target/jgit-4.9.target | 40 +++++++++---------- .../projects/jetty-9.4.x.tpd | 38 +++++++++--------- pom.xml | 2 +- 18 files changed, 335 insertions(+), 335 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index d9c407ab3..518e657ed 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -237,55 +237,55 @@ maven_jar( sha1 = "9180733b7df8542621dc12e21e87557e8c99b8cb", ) -JETTY_VER = "9.4.41.v20210516" +JETTY_VER = "9.4.42.v20210604" maven_jar( name = "jetty-servlet", artifact = "org.eclipse.jetty:jetty-servlet:" + JETTY_VER, - sha1 = "ea45368ea7fd04026038f89e6910f17f70939641", - src_sha1 = "4acf6b0d1449ccd39b195783e3639ab0da51f7bf", + sha1 = "2b0529a22eab56f9a5553dd963e79edee7ffc1b5", + src_sha1 = "3351f1fdde7a9bc214f42283af82f002666c777d", ) maven_jar( name = "jetty-security", artifact = "org.eclipse.jetty:jetty-security:" + JETTY_VER, - sha1 = "5ba69b1189a9d1f425ed03cbc2c901e0e6023c4d", - src_sha1 = "d46f8cb4dad66751d3a588309c6bbc15b80fbad4", + sha1 = "8f755242d4d73d98bef0055546aa53cf1ca456d9", + src_sha1 = "1442ba55bfdf147fdd9d8386627838b36af7a555", ) maven_jar( name = "jetty-server", artifact = "org.eclipse.jetty:jetty-server:" + JETTY_VER, - sha1 = "25b1963b0a1c56202ec37046adc55861815107ce", - src_sha1 = "a7f82c9df737316cf0dfafe4a33ca4ae89d780db", + sha1 = "f5f95cdabe677bd8aad9e80f5e125c5b1c5011aa", + src_sha1 = "bdf93b825e7a7478a4a47d1c7569fc468f8c949e", ) maven_jar( name = "jetty-http", artifact = "org.eclipse.jetty:jetty-http:" + JETTY_VER, - sha1 = "0d460bece4dd9666b46cbd18f8d7fd31cf02ecd9", - src_sha1 = "6fa009d950b8fdab8e94003e6295c08d42ee85b7", + sha1 = "2b0bb748d2388c6aa981ab29bc0a6a695dee31a4", + src_sha1 = "9f5b97a66ce62f03ce67b4f06936caa0ea9ba695", ) maven_jar( name = "jetty-io", artifact = "org.eclipse.jetty:jetty-io:" + JETTY_VER, - sha1 = "820eea368623939c2113902b1ca7a98186f64a73", - src_sha1 = "4373285dafb5f79210815d9c15de106cc3e9ac4d", + sha1 = "581c5bb1aca96934e6961315fff066000c783659", + src_sha1 = "6822c5df4877b9d631543dd3e048bb1f0d9c7249", ) maven_jar( name = "jetty-util", artifact = "org.eclipse.jetty:jetty-util:" + JETTY_VER, - sha1 = "548c76ea00d7eb3e2bcea273174e5d030639d109", - src_sha1 = "ba188de552a0c310f69cf12bea887413ce8f0e78", + sha1 = "16b116e9982c2037fd4bf05ce2525ccd5917eb9c", + src_sha1 = "e31bf18e992a5fd6e0e60f752b4edac4d57ca7cf", ) maven_jar( name = "jetty-util-ajax", artifact = "org.eclipse.jetty:jetty-util-ajax:" + JETTY_VER, - sha1 = "d4c1d66fc62796a17548e6c344fbf89b5889f873", - src_sha1 = "b60cf77be68137eee4ee13d83c47d684d14b6d90", + sha1 = "dfff709928e92d8fd4888d53dbdfb024a4aa92f9", + src_sha1 = "6b59220ec832e41ed433581409006a567691a75c", ) BOUNCYCASTLE_VER = "1.65" diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target index 8cc851d65..12dfd8f8f 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.10.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target index 164cd82ca..43831780c 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.11.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target index ed8ad0b6b..21bc9e672 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.12.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target index 993e1123a..cef88b13a 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.13.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target index e8cfa6073..de5e4dd50 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.14.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target index 81c64502b..583e32363 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.15.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target index 3afc3b8e6..3ccd87405 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.16.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target index f61261fc6..ac80f7f46 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.17.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target index c17c1c81a..cfee63015 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.18.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target index 6db7fe19f..a550bb4ff 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.19.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20-staging.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20-staging.target index f256622f4..c0324adad 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20-staging.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.20-staging.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target index ca1696be3..07e065519 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.6.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target index 60983880e..272ffc0c2 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.7.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target index 180b81452..367771488 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.8.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target index 27b3096f2..86fdad697 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/jgit-4.9.target @@ -1,28 +1,28 @@ - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd index fb0df1e8f..8645e9ba2 100644 --- a/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd +++ b/org.eclipse.jgit.packaging/org.eclipse.jgit.target/projects/jetty-9.4.x.tpd @@ -1,22 +1,22 @@ target "jetty-9.4.x" with source configurePhase -location jetty-9.4.40 "https://download.eclipse.org/jetty/updates/jetty-bundles-9.x/9.4.41.v20210516/" { - org.eclipse.jetty.client [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.client.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.continuation [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.continuation.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.http [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.http.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.io [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.io.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.security [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.security.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.server [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.server.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.servlet [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.servlet.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.util [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.util.source [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.util.ajax [9.4.41.v20210516,9.4.41.v20210516] - org.eclipse.jetty.util.ajax.source [9.4.41.v20210516,9.4.41.v20210516] +location jetty-9.4.40 "https://download.eclipse.org/jetty/updates/jetty-bundles-9.x/9.4.42.v20210604/" { + org.eclipse.jetty.client [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.client.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.continuation [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.continuation.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.http [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.http.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.io [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.io.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.security [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.security.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.server [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.server.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.servlet [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.servlet.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.util [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.util.source [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.util.ajax [9.4.42.v20210604,9.4.42.v20210604] + org.eclipse.jetty.util.ajax.source [9.4.42.v20210604,9.4.42.v20210604] } diff --git a/pom.xml b/pom.xml index b2e79058c..8cf38b7ed 100644 --- a/pom.xml +++ b/pom.xml @@ -162,7 +162,7 @@ 1.19 4.3.1 3.1.0 - 9.4.41.v20210516 + 9.4.42.v20210604 0.15.3 4.5.13 4.4.14 From fc57689774c5503838c96ee305f4977c585e87a2 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Thu, 17 Jun 2021 17:50:31 +0200 Subject: [PATCH 07/14] Fix PathSuffixFilter: can decide only on full paths On a subtree, a PathSuffixFilter must return -1 ("indeterminate"), not 0 ("include"), otherwise negation goes wrong: an indeterminate result (-1) is passed on, but a decision (0/1) is inverted. As a result a negated PathSuffixFilter would skip all folders. Bug: 574253 Change-Id: I27fe785c0d772392a5b5efe0a7b1c9cafcb6e566 Signed-off-by: Thomas Wolf --- .../eclipse/jgit/diff/DiffFormatterTest.java | 70 ++++++++++++++++++- .../treewalk/filter/PathSuffixFilterTest.java | 30 ++++++-- .../treewalk/filter/PathSuffixFilter.java | 11 ++- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java index 62824d3ae..b694f4aaf 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2020 Google Inc. and others + * Copyright (C) 2010, 2021 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -18,6 +18,8 @@ import java.io.BufferedOutputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.util.ArrayList; +import java.util.List; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.Status; @@ -34,8 +36,12 @@ import org.eclipse.jgit.patch.HunkHeader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.FileTreeIterator; +import org.eclipse.jgit.treewalk.filter.OrTreeFilter; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.eclipse.jgit.treewalk.filter.PathSuffixFilter; +import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.io.DisabledOutputStream; @@ -491,6 +497,68 @@ public void testTrackedFileInIgnoredFolderUnchanged() } } + @Test + public void testFilter() throws Exception { + RevCommit parent; + RevCommit head; + try (Git git = new Git(db)) { + writeTrashFile("foo.txt", "foo\n"); + writeTrashFile("src/some.txt", "some\n"); + writeTrashFile("src/image.png", "image\n"); + writeTrashFile("src/test.pdf", "test\n"); + writeTrashFile("src/xyz.txt", "xyz\n"); + git.add().addFilepattern(".").call(); + parent = git.commit().setMessage("initial").call(); + writeTrashFile("foo.txt", "FOO\n"); + writeTrashFile("src/some.txt", "SOME\n"); + writeTrashFile("src/image.png", "IMAGE\n"); + writeTrashFile("src/test.pdf", "TEST\n"); + writeTrashFile("src/xyz.txt", "XYZ\n"); + git.add().addFilepattern(".").call(); + head = git.commit().setMessage("second").call(); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + DiffFormatter dfmt = new DiffFormatter(os)) { + dfmt.setRepository(db); + List skip = new ArrayList<>(); + skip.add(PathSuffixFilter.create(".png")); + skip.add(PathSuffixFilter.create(".pdf")); + dfmt.setPathFilter(OrTreeFilter.create(skip).negate()); + dfmt.format( + new CanonicalTreeParser(null, db.newObjectReader(), + parent.getTree()), + new CanonicalTreeParser(null, db.newObjectReader(), + head.getTree())); + dfmt.flush(); + + String actual = os.toString("UTF-8"); + + String expected = "diff --git a/foo.txt b/foo.txt\n" + + "index 257cc56..b7d6715 100644\n" + + "--- a/foo.txt\n" + + "+++ b/foo.txt\n" + + "@@ -1 +1 @@\n" + + "-foo\n" + + "+FOO\n" + + "diff --git a/src/some.txt b/src/some.txt\n" + + "index 363ef61..76cea5f 100644\n" + + "--- a/src/some.txt\n" + + "+++ b/src/some.txt\n" + + "@@ -1 +1 @@\n" + + "-some\n" + + "+SOME\n" + + "diff --git a/src/xyz.txt b/src/xyz.txt\n" + + "index cd470e6..d4e3ab0 100644\n" + + "--- a/src/xyz.txt\n" + + "+++ b/src/xyz.txt\n" + + "@@ -1 +1 @@\n" + + "-xyz\n" + + "+XYZ\n"; + + assertEquals(expected, actual); + } + } + @Test public void testTrackedFileInIgnoredFolderChanged() throws Exception { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/filter/PathSuffixFilterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/filter/PathSuffixFilterTest.java index d8133913d..0b5a7356c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/filter/PathSuffixFilterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/filter/PathSuffixFilterTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, 2013 Google Inc. and others + * Copyright (C) 2009, 2021 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -41,10 +41,11 @@ public void testNonRecursiveFiltering() throws IOException { @Test public void testRecursiveFiltering() throws IOException { - ObjectId treeId = createTree("a.sth", "a.txt", "sub/b.sth", "sub/b.txt"); + ObjectId treeId = createTree("a.sth", "a.txt", "sub/b.sth", "sub/b.txt", + "t.sth", "t.txt"); List paths = getMatchingPaths(".txt", treeId, true); - List expected = Arrays.asList("a.txt", "sub/b.txt"); + List expected = Arrays.asList("a.txt", "sub/b.txt", "t.txt"); assertEquals(expected, paths); } @@ -59,6 +60,17 @@ public void testEdgeCases() throws IOException { assertEquals(Arrays.asList("abc", "c"), getMatchingPaths("c", treeId)); } + @Test + public void testNegated() throws IOException { + ObjectId treeId = createTree("a.sth", "a.txt", "sub/b.sth", + "sub/b.txt", "t.sth", "t.txt"); + + List paths = getMatchingPaths(".txt", treeId, true, true); + List expected = Arrays.asList("a.sth", "sub/b.sth", "t.sth"); + + assertEquals(expected, paths); + } + private ObjectId createTree(String... paths) throws IOException { final ObjectInserter odi = db.newObjectInserter(); final DirCache dc = db.readDirCache(); @@ -80,8 +92,18 @@ private List getMatchingPaths(String suffixFilter, private List getMatchingPaths(String suffixFilter, final ObjectId treeId, boolean recursiveWalk) throws IOException { + return getMatchingPaths(suffixFilter, treeId, recursiveWalk, false); + } + + private List getMatchingPaths(String suffixFilter, + final ObjectId treeId, boolean recursiveWalk, boolean negated) + throws IOException { try (TreeWalk tw = new TreeWalk(db)) { - tw.setFilter(PathSuffixFilter.create(suffixFilter)); + TreeFilter filter = PathSuffixFilter.create(suffixFilter); + if (negated) { + filter = filter.negate(); + } + tw.setFilter(filter); tw.setRecursive(recursiveWalk); tw.addTree(treeId); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java index 0c74bfdf6..3816d5ed0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/PathSuffixFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009, Google Inc. and others + * Copyright (C) 2009, 2021 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -73,6 +73,15 @@ public boolean include(TreeWalk walker) throws MissingObjectException, } + @Override + public int matchFilter(TreeWalk walker) throws MissingObjectException, + IncorrectObjectTypeException, IOException { + if (walker.isSubtree()) { + return -1; + } + return super.matchFilter(walker); + } + /** {@inheritDoc} */ @Override public boolean shouldBeRecursive() { From 1ae7e08bcae3a82c8cb201affae8136332753cb4 Mon Sep 17 00:00:00 2001 From: andrewxian2000 Date: Tue, 15 Jun 2021 09:58:52 +1200 Subject: [PATCH 08/14] Fix garbage collection failing to delete pack file The loosen() method has opened pack file and the open pack file handle may prevent it from being deleted e.g. on Windows. Fix this by closing the pack file only after loosen() finished. Bug: 574178 Change-Id: Icd59931a218d84c9c97b450eea87b21ed01248ff Signed-off-by: andrew.xian2000@gmail.com Signed-off-by: Matthias Sohn --- .../src/org/eclipse/jgit/internal/storage/file/GC.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 9ffff9f66..40c075ec5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -344,10 +344,10 @@ private void deleteOldPacks(Collection oldPacks, && repo.getFS() .lastModifiedInstant(oldPack.getPackFile()) .toEpochMilli() < packExpireDate) { - oldPack.close(); if (shouldLoosen) { loosen(inserter, reader, oldPack, ids); } + oldPack.close(); prunePack(oldPack.getPackFile()); } } From ffe55a80056552cbb8cb7c41349151d8d420e5b7 Mon Sep 17 00:00:00 2001 From: Marco Miller Date: Thu, 29 Apr 2021 14:44:48 -0400 Subject: [PATCH 09/14] Upgrade maven-jxr-plugin to 3.1.1 Change-Id: I66d7589ea1b823305d908acb13367ef78ef55f7a Signed-off-by: Marco Miller --- org.eclipse.jgit.benchmarks/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/org.eclipse.jgit.benchmarks/pom.xml b/org.eclipse.jgit.benchmarks/pom.xml index aeb2675ce..a2dcdb541 100644 --- a/org.eclipse.jgit.benchmarks/pom.xml +++ b/org.eclipse.jgit.benchmarks/pom.xml @@ -160,7 +160,7 @@ org.apache.maven.plugins maven-jxr-plugin - 3.0.0 + 3.1.1 org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index 8cf38b7ed..ad1bf938b 100644 --- a/pom.xml +++ b/pom.xml @@ -174,7 +174,7 @@ 1.65 4.2.3 3.1.2 - 3.0.0 + 3.1.1 3.0.0-M5 ${maven-surefire-plugin-version} 3.8.1 From 6692e394523cd7529002104dd4560de1f0fdd8e9 Mon Sep 17 00:00:00 2001 From: Marco Miller Date: Wed, 12 May 2021 08:50:29 -0400 Subject: [PATCH 10/14] Upgrade jacoco-maven-plugin to 0.8.7 Change-Id: I6742b2d8ec75422542a55a9840bfe43e8cf17400 Signed-off-by: Marco Miller --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ad1bf938b..6737f3ce9 100644 --- a/pom.xml +++ b/pom.xml @@ -345,7 +345,7 @@ org.jacoco jacoco-maven-plugin - 0.8.6 + 0.8.7 org.apache.maven.plugins From f598e69529e0a1864e8224265ed82326f2a296f5 Mon Sep 17 00:00:00 2001 From: Marco Miller Date: Thu, 17 Jun 2021 07:34:24 -0400 Subject: [PATCH 11/14] Upgrade maven-dependency-plugin to 3.2.0 Change-Id: I9dea176e8b30658bcb6a9dee8c54184c0fa86de8 Signed-off-by: Marco Miller --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6737f3ce9..226c7c723 100644 --- a/pom.xml +++ b/pom.xml @@ -255,7 +255,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.1.2 + 3.2.0 From 12f39c26b09bc3ebf1dd0216f56c37c808a53034 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 24 Jun 2021 23:13:51 +0200 Subject: [PATCH 12/14] Ignore missing javadoc in test bundles Change-Id: I83ed20823dc6b22ff48c2a554acb2f7d3b6067b7 --- org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs | 2 +- .../.settings/org.eclipse.jdt.core.prefs | 2 +- org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs | 2 +- .../.settings/org.eclipse.jdt.core.prefs | 2 +- org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs | 2 +- org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs | 2 +- .../.settings/org.eclipse.jdt.core.prefs | 2 +- .../.settings/org.eclipse.jdt.core.prefs | 2 +- org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..4c4634ada 100644 --- a/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.ant.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs index 822846c4d..d483c7794 100644 --- a/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.gpg.bc.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..4c4634ada 100644 --- a/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.http.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..4c4634ada 100644 --- a/org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..4c4634ada 100644 --- a/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.lfs.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..4c4634ada 100644 --- a/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.pgm.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs index 822846c4d..d483c7794 100644 --- a/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.ssh.apache.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs index 2bc2cf30d..6139f3899 100644 --- a/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.ssh.jsch.test/.settings/org.eclipse.jdt.core.prefs @@ -53,7 +53,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private diff --git a/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs index 3dd584039..4c4634ada 100644 --- a/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit.test/.settings/org.eclipse.jdt.core.prefs @@ -52,7 +52,7 @@ org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=protected org.eclipse.jdt.core.compiler.problem.missingJavadocTagDescription=return_tag -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=error +org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore org.eclipse.jdt.core.compiler.problem.missingJavadocTagsMethodTypeParameters=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=disabled org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=private From 24d6d605388c82201092cf1699b51095299380a2 Mon Sep 17 00:00:00 2001 From: Antonio Barone Date: Wed, 2 Jun 2021 18:13:17 +0300 Subject: [PATCH 13/14] Retry loose object read upon "Stale file handle" exception When reading loose objects over NFS it is possible that the OS syscall would fail with ESTALE errors: This happens when the open file descriptor no longer refers to a valid file. Notoriously it is possible to hit this scenario when git data is shared among multiple clients, for example by multiple gerrit instances in HA. If one of the two clients performs a GC operation that would cause the packing and then the pruning of loose objects, the other client might still hold a reference to those objects, which would cause an exception to bubble up the stack. The Linux NFS FAQ[1] (at point A.10), suggests that the proper way to handle such ESTALE scenarios is to: "[...] close the file or directory where the error occurred, and reopen it so the NFS client can resolve the pathname again and retrieve the new file handle." In case of a stale file handle exception, we now attempt to read the loose object again (up to 5 times), until we either succeed or encounter a FileNotFoundException, in which case the search can continue to Packfiles and alternates. The limit of 5 provides an arbitrary upper bounds that is consistent to the one chosen when handling stale file handles for packed-refs files (see [2] for context). [1] http://nfs.sourceforge.net/ [2] https://git.eclipse.org/r/c/jgit/jgit/+/54350 Bug: 573791 Change-Id: I9950002f772bbd8afeb9c6108391923be9d0ef51 --- .../storage/file/ObjectDirectoryTest.java | 41 +++++++++ .../eclipse/jgit/internal/JGitText.properties | 1 + .../org/eclipse/jgit/internal/JGitText.java | 1 + .../storage/file/ObjectDirectory.java | 86 +++++++++++++++---- 4 files changed, 112 insertions(+), 17 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java index d269457eb..a0bc63a69 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java @@ -44,8 +44,12 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import java.io.File; import java.io.IOException; @@ -69,6 +73,7 @@ import org.eclipse.jgit.util.FS; import org.junit.Assume; import org.junit.Test; +import org.mockito.Mockito; public class ObjectDirectoryTest extends RepositoryTestCase { @@ -194,6 +199,42 @@ public void testShallowFile() assertTrue(shallowCommits.isEmpty()); } + @Test + public void testOpenLooseObjectSuppressStaleFileHandleException() + throws Exception { + ObjectId id = ObjectId + .fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b"); + WindowCursor curs = new WindowCursor(db.getObjectDatabase()); + + ObjectDirectory mock = mock(ObjectDirectory.class); + UnpackedObjectCache unpackedObjectCacheMock = mock( + UnpackedObjectCache.class); + + Mockito.when(mock.getObjectLoader(any(), any(), any())) + .thenThrow(new IOException("Stale File Handle")); + Mockito.when(mock.openLooseObject(curs, id)).thenCallRealMethod(); + Mockito.when(mock.unpackedObjectCache()) + .thenReturn(unpackedObjectCacheMock); + + assertNull(mock.openLooseObject(curs, id)); + verify(unpackedObjectCacheMock).remove(id); + } + + @Test + public void testOpenLooseObjectPropagatesIOExceptions() throws Exception { + ObjectId id = ObjectId + .fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b"); + WindowCursor curs = new WindowCursor(db.getObjectDatabase()); + + ObjectDirectory mock = mock(ObjectDirectory.class); + + Mockito.when(mock.getObjectLoader(any(), any(), any())) + .thenThrow(new IOException("some IO failure")); + Mockito.when(mock.openLooseObject(curs, id)).thenCallRealMethod(); + + assertThrows(IOException.class, () -> mock.openLooseObject(curs, id)); + } + @Test public void testShallowFileCorrupt() throws Exception { FileRepository repository = createBareRepository(); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index c5e7dc3ef..2f3520de5 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -416,6 +416,7 @@ logInconsistentFiletimeDiff={}: inconsistent duration from file timestamps on {} logLargerFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: diff = {} > {} (last good value). Aborting measurement. logSmallerFiletime={}: got smaller file timestamp on {}, {}: {} < {}. Aborting measurement at resolution {}. logXDGConfigHomeInvalid=Environment variable XDG_CONFIG_HOME contains an invalid path {} +looseObjectHandleIsStale=loose-object {0} file handle is stale. retry {1} of {2} maxCountMustBeNonNegative=max count must be >= 0 mergeConflictOnNonNoteEntries=Merge conflict on non-note entries: base = {0}, ours = {1}, theirs = {2} mergeConflictOnNotes=Merge conflict on note {0}. base = {1}, ours = {2}, theirs = {2} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 814811d13..22200c26b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -444,6 +444,7 @@ public static JGitText get() { /***/ public String logLargerFiletimeDiff; /***/ public String logSmallerFiletime; /***/ public String logXDGConfigHomeInvalid; + /***/ public String looseObjectHandleIsStale; /***/ public String maxCountMustBeNonNegative; /***/ public String mergeConflictOnNonNoteEntries; /***/ public String mergeConflictOnNotes; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 265b71dd2..9d1d45736 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -85,6 +85,10 @@ public class ObjectDirectory extends FileObjectDatabase { /** Maximum number of candidates offered as resolutions of abbreviation. */ private static final int RESOLVE_ABBREV_LIMIT = 256; + /** Maximum number of attempts to read a loose object for which a stale file + * handle exception is thrown */ + final static int MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS = 5; + private final AlternateHandle handle = new AlternateHandle(this); private final Config config; @@ -212,7 +216,7 @@ public PackInserter newPackInserter() { /** {@inheritDoc} */ @Override public void close() { - unpackedObjectCache.clear(); + unpackedObjectCache().clear(); final PackList packs = packList.get(); if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { @@ -277,7 +281,7 @@ public String toString() { /** {@inheritDoc} */ @Override public boolean has(AnyObjectId objectId) { - return unpackedObjectCache.isUnpacked(objectId) + return unpackedObjectCache().isUnpacked(objectId) || hasPackedInSelfOrAlternate(objectId, null) || hasLooseInSelfOrAlternate(objectId, null); } @@ -395,7 +399,7 @@ private void resolve(Set matches, AbbreviatedObjectId id, @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { - if (unpackedObjectCache.isUnpacked(objectId)) { + if (unpackedObjectCache().isUnpacked(objectId)) { ObjectLoader ldr = openLooseObject(curs, objectId); if (ldr != null) { return ldr; @@ -473,23 +477,71 @@ ObjectLoader openPackedObject(WindowCursor curs, AnyObjectId objectId) { @Override ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id) throws IOException { - File path = fileFor(id); - try (FileInputStream in = new FileInputStream(path)) { - unpackedObjectCache.add(id); - return UnpackedObject.open(in, path, id, curs); - } catch (FileNotFoundException noFile) { - if (path.exists()) { - throw noFile; + int readAttempts = 0; + while (readAttempts < MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS) { + readAttempts++; + File path = fileFor(id); + try { + return getObjectLoader(curs, path, id); + } catch (FileNotFoundException noFile) { + if (path.exists()) { + throw noFile; + } + break; + } catch (IOException e) { + if (!FileUtils.isStaleFileHandleInCausalChain(e)) { + throw e; + } + if (LOG.isDebugEnabled()) { + LOG.debug(MessageFormat.format( + JGitText.get().looseObjectHandleIsStale, id.name(), + Integer.valueOf(readAttempts), Integer.valueOf( + MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS))); + } } - unpackedObjectCache.remove(id); - return null; } + unpackedObjectCache().remove(id); + return null; + } + + /** + * Provides a loader for an objectId + * + * @param curs + * cursor on the database + * @param path + * the path of the loose object + * @param id + * the object id + * @return a loader for the loose file object + * @throws IOException + * when file does not exist or it could not be opened + */ + ObjectLoader getObjectLoader(WindowCursor curs, File path, AnyObjectId id) + throws IOException { + try (FileInputStream in = new FileInputStream(path)) { + unpackedObjectCache().add(id); + return UnpackedObject.open(in, path, id, curs); + } + } + + /** + *

+ * Getter for the field unpackedObjectCache. + *

+ * This accessor is particularly useful to allow mocking of this class for + * testing purposes. + * + * @return the cache of the objects currently unpacked. + */ + UnpackedObjectCache unpackedObjectCache() { + return unpackedObjectCache; } @Override long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { - if (unpackedObjectCache.isUnpacked(id)) { + if (unpackedObjectCache().isUnpacked(id)) { long len = getLooseObjectSize(curs, id); if (0 <= len) { return len; @@ -567,13 +619,13 @@ private long getLooseObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { File f = fileFor(id); try (FileInputStream in = new FileInputStream(f)) { - unpackedObjectCache.add(id); + unpackedObjectCache().add(id); return UnpackedObject.getSize(in, id, curs); } catch (FileNotFoundException noFile) { if (f.exists()) { throw noFile; } - unpackedObjectCache.remove(id); + unpackedObjectCache().remove(id); return -1; } } @@ -667,7 +719,7 @@ InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { // If the object is already in the repository, remove temporary file. // - if (unpackedObjectCache.isUnpacked(id)) { + if (unpackedObjectCache().isUnpacked(id)) { FileUtils.delete(tmp, FileUtils.RETRY); return InsertLooseObjectResult.EXISTS_LOOSE; } @@ -723,7 +775,7 @@ private InsertLooseObjectResult tryMove(File tmp, File dst, Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst), StandardCopyOption.ATOMIC_MOVE); dst.setReadOnly(); - unpackedObjectCache.add(id); + unpackedObjectCache().add(id); return InsertLooseObjectResult.INSERTED; } From 6976a30f443ece4815a977b0a5a897c0236018f7 Mon Sep 17 00:00:00 2001 From: Fabio Ponciroli Date: Thu, 3 Jun 2021 16:15:17 +0200 Subject: [PATCH 14/14] searchForReuse might impact performance in large repositories The search for reuse phase for *all* the objects scans *all* the packfiles, looking for the best candidate to serve back to the client. This can lead to an expensive operation when the number of packfiles and objects is high. Add parameter "pack.searchForReuseTimeout" to limit the time spent on this search. Change-Id: I54f5cddb6796fdc93ad9585c2ab4b44854fa6c48 --- Documentation/config-options.md | 1 + .../internal/storage/file/PackWriterTest.java | 133 ++++++++++++++++++ org.eclipse.jgit/.settings/.api_filters | 8 ++ .../eclipse/jgit/internal/JGitText.properties | 1 + .../jgit/errors/SearchForReuseTimeout.java | 42 ++++++ .../org/eclipse/jgit/internal/JGitText.java | 1 + .../internal/storage/file/PackDirectory.java | 4 + .../internal/storage/pack/PackWriter.java | 44 ++++++ .../org/eclipse/jgit/lib/ConfigConstants.java | 8 ++ .../eclipse/jgit/storage/pack/PackConfig.java | 47 +++++++ .../eclipse/jgit/transport/UploadPack.java | 1 + 11 files changed, 290 insertions(+) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/errors/SearchForReuseTimeout.java diff --git a/Documentation/config-options.md b/Documentation/config-options.md index a9ca48c6a..19bcc3352 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -102,6 +102,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `prunePreserved`, only via API of PackConfig | `false` | ⃞ | Whether to remove preserved pack files in a preserved directory. | | `pack.reuseDeltas` | `true` |⃞ | Whether to reuse deltas existing in repository. | | `pack.reuseObjects` | `true` | ⃞ | Whether to reuse existing objects representation in repository. | +| `pack.searchForReuseTimeout` | | ⃞ | Search for reuse phase timeout. Expressed as a `Duration`, i.e.: `50sec`. | | `pack.singlePack` | `false` | ⃞ | Whether all of `refs/*` should be packed in a single pack. | | `pack.threads` | `0` (auto-detect number of processors) | ✅ | Number of threads to use for delta compression. | | `pack.waitPreventRacyPack` | `false` | ⃞ | Whether we wait before opening a newly written pack to prevent its lastModified timestamp could be racy. | diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java index e422ab9db..71aca9d80 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java @@ -18,6 +18,10 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -25,6 +29,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.text.ParseException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -32,6 +37,7 @@ import java.util.List; import java.util.Set; +import org.eclipse.jgit.api.Git; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -43,6 +49,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdSet; import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.Sets; import org.eclipse.jgit.revwalk.DepthWalk; @@ -58,6 +65,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; public class PackWriterTest extends SampleDataRepositoryTestCase { @@ -626,6 +634,131 @@ public void testShallowFetchShallowAncestorDepth2() throws Exception { } } + @Test + public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet() + throws Exception { + FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackWriter mockedPackWriter = Mockito + .spy(new PackWriter(config, fileRepository.newObjectReader())); + + doNothing().when(mockedPackWriter).select(any(), any()); + + try (FileOutputStream packOS = new FileOutputStream( + getPackFileToWrite(fileRepository, mockedPackWriter))) { + mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + + long numberOfPackFiles = new GC(fileRepository) + .getStatistics().numberOfPackFiles; + int expectedSelectCalls = + // Objects contained in multiple packfiles * number of packfiles + 2 * (int) numberOfPackFiles + + // Objects in single packfile + 1; + verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), + any()); + } + + @Test + public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck() + throws Exception { + FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackConfig packConfig = new PackConfig(); + packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); + PackWriter mockedPackWriter = Mockito.spy( + new PackWriter(packConfig, fileRepository.newObjectReader())); + + doNothing().when(mockedPackWriter).select(any(), any()); + + try (FileOutputStream packOS = new FileOutputStream( + getPackFileToWrite(fileRepository, mockedPackWriter))) { + mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + + long numberOfPackFiles = new GC(fileRepository) + .getStatistics().numberOfPackFiles; + int expectedSelectCalls = + // Objects contained in multiple packfiles * number of packfiles + 2 * (int) numberOfPackFiles + + // Objects contained in single packfile + 1; + verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), + any()); + } + + @Test + public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck() + throws Exception { + FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackConfig packConfig = new PackConfig(); + packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); + PackWriter mockedPackWriter = Mockito.spy( + new PackWriter(packConfig, fileRepository.newObjectReader())); + mockedPackWriter.enableSearchForReuseTimeout(); + + doNothing().when(mockedPackWriter).select(any(), any()); + + try (FileOutputStream packOS = new FileOutputStream( + getPackFileToWrite(fileRepository, mockedPackWriter))) { + mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + + int expectedSelectCalls = 3; // Objects in packfiles + verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), + any()); + } + + /** + * Creates objects and packfiles in the following order: + *
    + *
  • Creates 2 objects (C1 = commit, T1 = tree) + *
  • Creates packfile P1 (containing C1, T1) + *
  • Creates 1 object (C2 commit) + *
  • Creates packfile P2 (containing C1, T1, C2) + *
  • Create 1 object (C3 commit) + *
+ * + * @throws Exception + */ + private FileRepository setUpRepoWithMultiplePackfiles() throws Exception { + FileRepository fileRepository = createWorkRepository(); + try (Git git = new Git(fileRepository)) { + // Creates 2 objects (C1 = commit, T1 = tree) + git.commit().setMessage("First commit").call(); + GC gc = new GC(fileRepository); + gc.setPackExpireAgeMillis(Long.MAX_VALUE); + gc.setExpireAgeMillis(Long.MAX_VALUE); + // Creates packfile P1 (containing C1, T1) + gc.gc(); + // Creates 1 object (C2 commit) + git.commit().setMessage("Second commit").call(); + // Creates packfile P2 (containing C1, T1, C2) + gc.gc(); + // Create 1 object (C3 commit) + git.commit().setMessage("Third commit").call(); + } + return fileRepository; + } + + private PackFile getPackFileToWrite(FileRepository fileRepository, + PackWriter mockedPackWriter) throws IOException { + File packdir = fileRepository.getObjectDatabase().getPackDirectory(); + PackFile packFile = new PackFile(packdir, + mockedPackWriter.computeName(), PackExt.PACK); + + Set all = new HashSet<>(); + for (Ref r : fileRepository.getRefDatabase().getRefs()) { + all.add(r.getObjectId()); + } + + mockedPackWriter.preparePack(NullProgressMonitor.INSTANCE, all, + PackWriter.NONE); + return packFile; + } + private FileRepository setupRepoForShallowFetch() throws Exception { FileRepository repo = createBareRepository(); // TestRepository will close the repo, but we need to return an open diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 33331fbab..14c505de0 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,5 +1,13 @@ + + + + + + + + diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 962324e0f..848d20aba 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -632,6 +632,7 @@ s3ActionWriting=Writing searchForReachableBranches=Finding reachable branches saveFileStoreAttributesFailed=Saving measured FileStore attributes to user config failed searchForReuse=Finding sources +searchForReuseTimeout=Search for reuse timed out after {0} seconds searchForSizes=Getting sizes secondsAgo={0} seconds ago selectingCommits=Selecting commits diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/SearchForReuseTimeout.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/SearchForReuseTimeout.java new file mode 100644 index 000000000..402a85061 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/SearchForReuseTimeout.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2021, Fabio Ponciroli + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.errors; + +import org.eclipse.jgit.internal.JGitText; + +import java.io.IOException; +import java.text.MessageFormat; +import java.time.Duration; + +/** + * Thrown when the search for reuse phase times out. + * + * @since 5.13 + */ +public class SearchForReuseTimeout extends IOException { + private static final long serialVersionUID = 1L; + + /** + * Construct a search for reuse timeout error. + * + * @param timeout + * time exceeded during the search for reuse phase. + */ + public SearchForReuseTimeout(Duration timeout) { + super(MessageFormat.format(JGitText.get().searchForReuseTimeout, + Long.valueOf(timeout.getSeconds()))); + } + + @Override + public synchronized Throwable fillInStackTrace() { + return this; + } +} \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index fd54986f9..46d96df9c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -660,6 +660,7 @@ public static JGitText get() { /***/ public String saveFileStoreAttributesFailed; /***/ public String searchForReachableBranches; /***/ public String searchForReuse; + /***/ public String searchForReuseTimeout; /***/ public String searchForSizes; /***/ public String secondsAgo; /***/ public String selectingCommits; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 73745d8c6..f32909f44 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -33,6 +33,7 @@ import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -264,7 +265,10 @@ void selectRepresentation(PackWriter packer, ObjectToPack otp, p.resetTransientErrorCount(); if (rep != null) { packer.select(otp, rep); + packer.checkSearchForReuseTimeout(); } + } catch (SearchForReuseTimeout e) { + break SEARCH; } catch (PackMismatchException e) { // Pack was modified; refresh the entire pack list. // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 3e4b5df6a..61f92d2e1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -25,6 +25,7 @@ import java.lang.ref.WeakReference; import java.security.MessageDigest; import java.text.MessageFormat; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -54,6 +55,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; @@ -262,6 +264,12 @@ public static Iterable getInstances() { private boolean indexDisabled; + private boolean checkSearchForReuseTimeout = false; + + private final Duration searchForReuseTimeout; + + private long searchForReuseStartTimeEpoc; + private int depth; private Collection unshallowObjects; @@ -356,6 +364,7 @@ public PackWriter(PackConfig config, final ObjectReader reader, deltaBaseAsOffset = config.isDeltaBaseAsOffset(); reuseDeltas = config.isReuseDeltas(); + searchForReuseTimeout = config.getSearchForReuseTimeout(); reuseValidate = true; // be paranoid by default stats = statsAccumulator != null ? statsAccumulator : new PackStatistics.Accumulator(); @@ -404,6 +413,24 @@ public boolean isDeltaBaseAsOffset() { return deltaBaseAsOffset; } + /** + * Check whether the search for reuse phase is taking too long. This could + * be the case when the number of objects and pack files is high and the + * system is under pressure. If that's the case and + * checkSearchForReuseTimeout is true abort the search. + * + * @throws SearchForReuseTimeout + * if the search for reuse is taking too long. + */ + public void checkSearchForReuseTimeout() throws SearchForReuseTimeout { + if (checkSearchForReuseTimeout + && Duration.ofMillis(System.currentTimeMillis() + - searchForReuseStartTimeEpoc) + .compareTo(searchForReuseTimeout) > 0) { + throw new SearchForReuseTimeout(searchForReuseTimeout); + } + } + /** * Set writer delta base format. Delta base can be written as an offset in a * pack file (new approach reducing file size) or as an object id (legacy @@ -419,6 +446,22 @@ public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) { this.deltaBaseAsOffset = deltaBaseAsOffset; } + /** + * Set the writer to check for long search for reuse, exceeding the timeout. + * Selecting an object representation can be an expensive operation. It is + * possible to set a max search for reuse time (see + * PackConfig#CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT for more details). + * + * However some operations, i.e.: GC, need to find the best candidate + * regardless how much time the operation will need to finish. + * + * This method enables the search for reuse timeout check, otherwise + * disabled. + */ + public void enableSearchForReuseTimeout() { + this.checkSearchForReuseTimeout = true; + } + /** * Check if the writer will reuse commits that are already stored as deltas. * @@ -1306,6 +1349,7 @@ private void searchForReuse(ProgressMonitor monitor) throws IOException { cnt += objectsLists[OBJ_TAG].size(); long start = System.currentTimeMillis(); + searchForReuseStartTimeEpoc = start; beginPhase(PackingPhase.FINDING_SOURCES, monitor, cnt); if (cnt <= 4096) { // For small object counts, do everything as one list. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 3e3d9b569..b6f4798dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -736,4 +736,12 @@ public final class ConfigConstants { * @since 5.11 */ public static final String CONFIG_KEY_DEFAULT_BRANCH = "defaultbranch"; + + /** + * The "pack.searchForReuseTimeout" key + * + * @since 5.13 + */ + public static final String CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT = "searchforreusetimeout"; + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index f76dd2721..6aa8be642 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -29,6 +29,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_DELTAS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_OBJECTS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SINGLE_PACK; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_THREADS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK; @@ -36,7 +37,9 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; +import java.time.Duration; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; import java.util.zip.Deflater; import org.eclipse.jgit.internal.storage.file.PackIndexWriter; @@ -222,6 +225,16 @@ public class PackConfig { */ public static final int DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS = 90; + /** + * Default max time to spend during the search for reuse phase. This + * optimization is disabled by default: {@value} + * + * @see #setSearchForReuseTimeout(Duration) + * @since 5.13 + */ + public static final Duration DEFAULT_SEARCH_FOR_REUSE_TIMEOUT = Duration + .ofSeconds(Integer.MAX_VALUE); + private int compressionLevel = Deflater.DEFAULT_COMPRESSION; private boolean reuseDeltas = DEFAULT_REUSE_DELTAS; @@ -272,6 +285,8 @@ public class PackConfig { private int bitmapInactiveBranchAgeInDays = DEFAULT_BITMAP_INACTIVE_BRANCH_AGE_IN_DAYS; + private Duration searchForReuseTimeout = DEFAULT_SEARCH_FOR_REUSE_TIMEOUT; + private boolean cutDeltaChains; private boolean singlePack; @@ -342,6 +357,7 @@ public PackConfig(PackConfig cfg) { this.bitmapInactiveBranchAgeInDays = cfg.bitmapInactiveBranchAgeInDays; this.cutDeltaChains = cfg.cutDeltaChains; this.singlePack = cfg.singlePack; + this.searchForReuseTimeout = cfg.searchForReuseTimeout; } /** @@ -1103,6 +1119,18 @@ public int getBitmapInactiveBranchAgeInDays() { return bitmapInactiveBranchAgeInDays; } + /** + * Get the max time to spend during the search for reuse phase. + * + * Default setting: {@value #DEFAULT_SEARCH_FOR_REUSE_TIMEOUT} + * + * @return the maximum time to spend during the search for reuse phase. + * @since 5.13 + */ + public Duration getSearchForReuseTimeout() { + return searchForReuseTimeout; + } + /** * Set the age in days that marks a branch as "inactive". * @@ -1116,6 +1144,19 @@ public void setBitmapInactiveBranchAgeInDays(int ageInDays) { bitmapInactiveBranchAgeInDays = ageInDays; } + /** + * Set the max time to spend during the search for reuse phase. + * + * @param timeout + * max time allowed during the search for reuse phase + * + * Default setting: {@value #DEFAULT_SEARCH_FOR_REUSE_TIMEOUT} + * @since 5.13 + */ + public void setSearchForReuseTimeout(Duration timeout) { + searchForReuseTimeout = timeout; + } + /** * Update properties by setting fields from the configuration. * @@ -1179,6 +1220,10 @@ public void fromConfig(Config rc) { setBitmapInactiveBranchAgeInDays(rc.getInt(CONFIG_PACK_SECTION, CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS, getBitmapInactiveBranchAgeInDays())); + setSearchForReuseTimeout(Duration.ofSeconds(rc.getTimeUnit( + CONFIG_PACK_SECTION, null, + CONFIG_KEY_SEARCH_FOR_REUSE_TIMEOUT, + getSearchForReuseTimeout().getSeconds(), TimeUnit.SECONDS))); setWaitPreventRacyPack(rc.getBoolean(CONFIG_PACK_SECTION, CONFIG_KEY_WAIT_PREVENT_RACYPACK, isWaitPreventRacyPack())); setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION, @@ -1216,6 +1261,8 @@ public String toString() { .append(getBitmapExcessiveBranchCount()); b.append(", bitmapInactiveBranchAge=") //$NON-NLS-1$ .append(getBitmapInactiveBranchAgeInDays()); + b.append(", searchForReuseTimeout") //$NON-NLS-1$ + .append(getSearchForReuseTimeout()); b.append(", singlePack=").append(getSinglePack()); //$NON-NLS-1$ return b.toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index ecf175193..37a1c1e58 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -2359,6 +2359,7 @@ else if (ref.getName().startsWith(Constants.R_HEADS)) GitProtocolConstants.SECTION_PACKFILE + '\n'); } } + pw.enableSearchForReuseTimeout(); pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); if (msgOut != NullOutputStream.INSTANCE) {