@@ -961,6 +966,9 @@ ant test -Dtests.iters=N -Dtestcase=ClassName -Dtests.seed=dead:beef
ant test -Dtests.iters=N -Dtestcase=ClassName -Dtestmethod=mytest
ant test -Dtests.iters=N -Dtestcase=ClassName -Dtests.method=mytest*
+# Repeats N times but skips any tests after hitting X failures
+ant test -Dtests.iters=N -Dtests.maxfailures=X -Dtestcase=...
+
#
# Test groups. ----------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java b/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
new file mode 100644
index 0000000..67e8a55
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
@@ -0,0 +1,72 @@
+package org.apache.lucene.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.junitcompat.WithNestedTests;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.*;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
+
+/**
+ * @see TestRuleIgnoreAfterMaxFailures
+ * @see SystemPropertiesInvariantRule
+ */
+public class TestMaxFailuresRule extends WithNestedTests {
+ public TestMaxFailuresRule() {
+ super(true);
+ }
+
+ public static class Nested extends WithNestedTests.AbstractNestedTest {
+ @Repeat(iterations = 100)
+ public void testFailSometimes() {
+ assertFalse(random().nextInt(5) == 0);
+ }
+ }
+
+ @Test
+ public void testMaxFailures() {
+ int maxFailures = LuceneTestCase.ignoreAfterMaxFailures.getMaxFailures();
+ try {
+ LuceneTestCase.ignoreAfterMaxFailures.setMaxFailures(2);
+
+ JUnitCore core = new JUnitCore();
+ final int [] assumptions = new int [1];
+ core.addListener(new RunListener() {
+ @Override
+ public void testAssumptionFailure(Failure failure) {
+ assumptions[0]++;
+ }
+ });
+
+ Result result = core.run(Nested.class);
+ Assert.assertEquals(2, result.getFailureCount());
+ Assert.assertEquals(0, result.getIgnoreCount());
+ Assert.assertEquals(100, result.getRunCount());
+ // JUnit doesn't pass back the number of successful tests, just make sure
+ // we did have enough assumption-failures.
+ Assert.assertTrue(assumptions[0] > 50);
+ } finally {
+ LuceneTestCase.ignoreAfterMaxFailures.setMaxFailures(maxFailures);
+ }
+ }
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 39320c8..bd16587 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -123,6 +123,7 @@ public abstract class LuceneTestCase extends Assert {
public static final String SYSPROP_WEEKLY = "tests.weekly";
public static final String SYSPROP_AWAITSFIX = "tests.awaitsfix";
public static final String SYSPROP_SLOW = "tests.slow";
+ public static final String SYSPROP_MAXFAILURES = "tests.maxfailures";
/**
* Annotation for tests that should only be run during nightly builds.
@@ -298,8 +299,16 @@ public abstract class LuceneTestCase extends Assert {
/**
* Suite failure marker (any error in the test or suite scope).
*/
- public static TestRuleMarkFailure suiteFailureMarker;
-
+ public final static TestRuleMarkFailure suiteFailureMarker =
+ new TestRuleMarkFailure();
+
+ /**
+ * Ignore tests after hitting a designated number of initial failures.
+ */
+ final static TestRuleIgnoreAfterMaxFailures ignoreAfterMaxFailures =
+ new TestRuleIgnoreAfterMaxFailures(
+ RandomizedTest.systemPropertyAsInt(SYSPROP_MAXFAILURES, Integer.MAX_VALUE));
+
/**
* This controls how suite-level rules are nested. It is important that _all_ rules declared
* in {@link LuceneTestCase} are executed in proper order if they depend on each
@@ -308,7 +317,8 @@ public abstract class LuceneTestCase extends Assert {
@ClassRule
public static TestRule classRules = RuleChain
.outerRule(new TestRuleIgnoreTestSuites())
- .around(suiteFailureMarker = new TestRuleMarkFailure())
+ .around(ignoreAfterMaxFailures)
+ .around(suiteFailureMarker)
.around(new TestRuleAssertionsRequired())
.around(new TestRuleNoStaticHooksShadowing())
.around(new TestRuleNoInstanceHooksOverrides())
@@ -329,7 +339,7 @@ public abstract class LuceneTestCase extends Assert {
/** Save test thread and name. */
private TestRuleThreadAndTestName threadAndTestNameRule = new TestRuleThreadAndTestName();
- /** Taint test failures. */
+ /** Taint suite result with individual test failures. */
private TestRuleMarkFailure testFailureMarker = new TestRuleMarkFailure(suiteFailureMarker);
/**
@@ -340,6 +350,7 @@ public abstract class LuceneTestCase extends Assert {
@Rule
public final TestRule ruleChain = RuleChain
.outerRule(testFailureMarker)
+ .around(ignoreAfterMaxFailures)
.around(threadAndTestNameRule)
.around(new TestRuleReportUncaughtExceptions())
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java
new file mode 100644
index 0000000..aca4177
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java
@@ -0,0 +1,90 @@
+package org.apache.lucene.util;
+
+import org.junit.Assert;
+import org.junit.internal.AssumptionViolatedException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This rule keeps a count of failed tests (suites) and will result in an
+ * {@link AssumptionViolatedException} after a given number of failures for all
+ * tests following this condition.
+ *
+ *
+ * Aborting quickly on failed tests can be useful when used in combination with
+ * test repeats (via the {@link Repeat} annotation or system property).
+ */
+public final class TestRuleIgnoreAfterMaxFailures implements TestRule {
+ /**
+ * Maximum failures.
+ */
+ private int maxFailures;
+
+ /**
+ * Current count of failures.
+ */
+ private int failuresSoFar;
+
+ /**
+ * @param maxFailures
+ * The number of failures after which all tests are ignored. Must be
+ * greater or equal 1.
+ */
+ public TestRuleIgnoreAfterMaxFailures(int maxFailures) {
+ Assert.assertTrue("maxFailures must be >= 1: " + maxFailures, maxFailures >= 1);
+ this.maxFailures = maxFailures;
+ }
+
+ @Override
+ public Statement apply(final Statement s, final Description d) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ if (failuresSoFar >= maxFailures) {
+ RandomizedTest.assumeTrue("Ignored, failures limit reached (" +
+ failuresSoFar + " >= " + maxFailures + ").", false);
+ }
+
+ try {
+ s.evaluate();
+ } catch (Throwable t) {
+ if (!TestRuleMarkFailure.isAssumption(t)) {
+ System.out.println("#" + d);
+ failuresSoFar++;
+ }
+ throw t;
+ }
+ }
+ };
+ }
+
+ /** For tests only. */
+ void setMaxFailures(int maxFailures) {
+ this.maxFailures = maxFailures;
+ }
+
+ int getMaxFailures() {
+ return maxFailures;
+ }
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java
index 22a8e3f..6020904 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreTestSuites.java
@@ -1,6 +1,5 @@
package org.apache.lucene.util;
-import org.junit.Assume;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java
index d400445..be9c625 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleMarkFailure.java
@@ -47,11 +47,8 @@ public final class TestRuleMarkFailure implements TestRule {
try {
s.evaluate();
} catch (Throwable t) {
- for (Throwable t2 : expandFromMultiple(t)) {
- if (!(t2 instanceof AssumptionViolatedException)) {
- markFailed();
- break;
- }
+ if (!isAssumption(t)) {
+ markFailed();
}
throw t;
}
@@ -60,6 +57,19 @@ public final class TestRuleMarkFailure implements TestRule {
}
/**
+ * Is a given exception (or a MultipleFailureException) an
+ * {@link AssumptionViolatedException}?
+ */
+ public static boolean isAssumption(Throwable t) {
+ for (Throwable t2 : expandFromMultiple(t)) {
+ if (!(t2 instanceof AssumptionViolatedException)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
* Expand from multi-exception wrappers.
*/
private static List expandFromMultiple(Throwable t) {