From 2d4d64481b397852a21813defe40ff70c3e79718 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 25 May 2016 01:07:18 +0200 Subject: [PATCH] Add a RepeatRule to help repeating flaky tests A JUnit TestRule which enables to run the same JUnit test repeatedly. This may help to identify the root cause why a flaky tests which succeed most often does fail sometimes. Add the RepeatRule to the test class containing the test to be repeated: public class MyTest { @Rule public RepeatRule repeatRule = new RepeatRule(); ... } and annotate the test to be repeated with the @Repeat(n=) annotation: @Test @Repeat(n = 100) public void test() { ... } then this test will be repeated 100 times. If any test execution fails test repetition will be stopped. Change-Id: I7c49ccebe1cb00bcde6b002b522d95c13fd3a35e Signed-off-by: Matthias Sohn --- org.eclipse.jgit.junit/META-INF/MANIFEST.MF | 5 +- .../src/org/eclipse/jgit/junit/Repeat.java | 53 +++++++ .../org/eclipse/jgit/junit/RepeatRule.java | 131 ++++++++++++++++++ 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java create mode 100644 org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java diff --git a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF index 8a010fc56..69cbb17f8 100644 --- a/org.eclipse.jgit.junit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.junit/META-INF/MANIFEST.MF @@ -21,7 +21,10 @@ Import-Package: org.eclipse.jgit.api;version="[4.5.0,4.6.0)", org.eclipse.jgit.treewalk.filter;version="[4.5.0,4.6.0)", org.eclipse.jgit.util;version="[4.5.0,4.6.0)", org.eclipse.jgit.util.io;version="[4.5.0,4.6.0)", - org.junit;version="[4.0.0,5.0.0)" + org.junit;version="[4.0.0,5.0.0)", + org.junit.rules;version="[4.9.0,5.0.0)", + org.junit.runner;version="[4.0.0,5.0.0)", + org.junit.runners.model;version="[4.5.0,5.0.0)" Export-Package: org.eclipse.jgit.junit;version="4.5.0"; uses:="org.eclipse.jgit.dircache, org.eclipse.jgit.lib, diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java new file mode 100644 index 000000000..22b5007fb --- /dev/null +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2016, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.junit; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ java.lang.annotation.ElementType.METHOD }) +public @interface Repeat { + public abstract int n(); +} \ No newline at end of file diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java new file mode 100644 index 000000000..75e1a67a5 --- /dev/null +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2016, Matthias Sohn + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.junit; + +import java.text.MessageFormat; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +/** + * {@link TestRule} which enables to run the same JUnit test repeatedly. Add + * this rule to the test class + * + * 
+ * public class MyTest {
+ * 	@Rule
+ * 	public RepeatRule repeatRule = new RepeatRule();
+ * 	...
+ * }
+ *
+ * + * and annotate the test to be repeated with the + * {@code @Repeat(n=)} annotation + * + * 
+ * @Test
+ * @Repeat(n = 100)
+ * public void test() {
+ * 	...
+ * }
+ *
+ * + * then this test will be repeated 100 times. If any test execution fails test + * repetition will be stopped. + */ +public class RepeatRule implements TestRule { + + private static Logger LOG = Logger + .getLogger(RepeatRule.class.getName()); + + public static class RepeatedTestException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public RepeatedTestException(String message, Throwable cause) { + super(message, cause); + } + } + + private static class RepeatStatement extends Statement { + + private final int repetitions; + + private final Statement statement; + + private RepeatStatement(int repetitions, Statement statement) { + this.repetitions = repetitions; + this.statement = statement; + } + + @Override + public void evaluate() throws Throwable { + for (int i = 0; i < repetitions; i++) { + try { + statement.evaluate(); + } catch (Throwable e) { + RepeatedTestException ex = new RepeatedTestException( + MessageFormat.format( + "Repeated test failed when run for the {0}. time", + Integer.valueOf(i + 1)), + e); + LOG.log(Level.SEVERE, ex.getMessage(), ex); + throw ex; + } + } + } + } + + @Override + public Statement apply(Statement statement, Description description) { + Statement result = statement; + Repeat repeat = description.getAnnotation(Repeat.class); + if (repeat != null) { + int n = repeat.n(); + result = new RepeatStatement(n, statement); + } + return result; + } +} \ No newline at end of file