diff --git .travis.yml .travis.yml new file mode 100644 index 0000000..2200591 --- /dev/null +++ .travis.yml @@ -0,0 +1,27 @@ +language: java +before_install: + - mvn -v + - ps axu +install: + - mvn clean install -DskipTests +cache: + directories: + - $HOME/.m2 +env: + - BUILD="test -PskipServerTests" + - BUILD="-pl hbase-server test -PrunFlakeyTests" + - BUILD="-pl hbase-server test -PrunVerySlowRegionServerTests" + - BUILD="-pl hbase-server test -PrunRegionServerTests" + - BUILD="-pl hbase-server test -PrunMiscTests" + - BUILD="-pl hbase-server test -PrunClientTests" + - BUILD="-pl hbase-server test -PrunCoprocessorTests" + - BUILD="-pl hbase-server test -PrunFilterTests" + - BUILD="-pl hbase-server test -PrunMasterTests" + - BUILD="-pl hbase-server test -PrunIOTests" + - BUILD="-pl hbase-server test -PrunRPCTests" + - BUILD="-pl hbase-server test -PrunRestTests" + - BUILD="-pl hbase-server test -PrunReplicationTests" + - BUILD="-pl hbase-server test -PrunSecurityTests" + +script: + - set -x; mvn -DreuseForks=true -Dsurefire.timeout=9000 ${BUILD} diff --git circle.yml circle.yml new file mode 100644 index 0000000..a2dca1f --- /dev/null +++ circle.yml @@ -0,0 +1,13 @@ +machine: + java: + version: oraclejdk7 +dependencies: + override: + - mvn clean install -DskipTests + cache_directories: + - ~/.m2 +test: + override: + - set -x ; dev-support/split_across_ci.sh: + parallel: true + timeout: 1800 diff --git dev-support/split_across_ci.sh dev-support/split_across_ci.sh new file mode 100755 index 0000000..3ccd102 --- /dev/null +++ dev-support/split_across_ci.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -ex + +i=0 +tests='echo starting' +for test in "test -PskipServerTests" \ +"-pl hbase-server test -PrunFlakeyTests" \ +"-pl hbase-server test -PrunVerySlowRegionServerTests" \ +"-pl hbase-server test -PrunRegionServerTests" \ +"-pl hbase-server test -PrunMiscTests" \ +"-pl hbase-server test -PrunClientTests" \ +"-pl hbase-server test -PrunMasterTests" \ +"-pl hbase-server test -PrunCoprocessorTests" \ +"-pl hbase-server test -PrunFilterTests" \ +"-pl hbase-server test -PrunIOTests" \ +"-pl hbase-server test -PrunRPCTests" \ +"-pl hbase-server test -PrunRestTests" \ +"-pl hbase-server test -PrunReplicationTests" \ +"-pl hbase-server test -PrunSecurityTests" \ + +do + if [ $(($i % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX ] + then + tests+=&&mvn ${test} + fi + ((i++))|| ps ax +done + +${tests} diff --git hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RetryRunner.java hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RetryRunner.java new file mode 100644 index 0000000..c1b3d54 --- /dev/null +++ hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/RetryRunner.java @@ -0,0 +1,102 @@ +package org.apache.hadoop.hbase.testclassification; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.internal.AssumptionViolatedException; +import org.junit.internal.runners.model.EachTestNotifier; +import org.junit.runner.Description; +import org.junit.runner.notification.RunNotifier; +import org.junit.runner.notification.StoppedByUserException; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; +import org.junit.runners.model.TestClass; + +import java.util.List; + + +public class RetryRunner extends BlockJUnit4ClassRunner { + + private final int retryCount = 10; + private final TestClass testClassToRetry; + private int failedAttempts = 0; + + public RetryRunner(Class klass) throws InitializationError { + super(klass); + this.testClassToRetry = new TestClass(klass); + } + + + @Override + public void run(final RunNotifier notifier) { + EachTestNotifier testNotifier = new EachTestNotifier(notifier, + getDescription()); + Statement statement = classBlock(notifier); + try { + statement.evaluate(); + } catch (AssumptionViolatedException e) { + testNotifier.fireTestIgnored(); + } catch (StoppedByUserException e) { + throw e; + } catch (Throwable e) { + retry(testNotifier, statement, e); + } + } + + @Override + protected void runChild(final FrameworkMethod method, RunNotifier notifier) { + Description description = describeChild(method); + if (method.getAnnotation(Ignore.class) != null) { + notifier.fireTestIgnored(description); + } else { + runTestUnit(methodBlock(method), description, notifier); + } + } + + /** + * Runs a {@link Statement} that represents a leaf (aka atomic) test. + */ + protected final void runTestUnit(Statement statement, Description description, + RunNotifier notifier) { + EachTestNotifier eachNotifier = new EachTestNotifier(notifier, description); + eachNotifier.fireTestStarted(); + try { + statement.evaluate(); + } catch (AssumptionViolatedException e) { + eachNotifier.addFailedAssumption(e); + } catch (Throwable e) { + retry(eachNotifier, statement, e); + } finally { + eachNotifier.fireTestFinished(); + } + } + + public void retry(EachTestNotifier notifier, Statement statement, Throwable currentThrowable) { + Throwable caughtThrowable = currentThrowable; + while (retryCount > failedAttempts) { + try { + // Rerun all of the afterclasses + List afterCs = testClassToRetry.getAnnotatedMethods(AfterClass.class); + for (FrameworkMethod after: afterCs){ + after.invokeExplosively(testClassToRetry); + } + + // Rerun all of the beforeclasses + List beforCs = testClassToRetry.getAnnotatedMethods(BeforeClass.class); + for (FrameworkMethod befor: beforCs){ + befor.invokeExplosively(testClassToRetry); + } + + statement.evaluate(); + return; + } catch (Throwable t) { + failedAttempts++; + caughtThrowable = t; + } + } + notifier.addFailure(caughtThrowable); + } +} \ No newline at end of file diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index b4e94bf..491a4ba 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -133,8 +133,8 @@ import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.FlakeyTests; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; import org.apache.hadoop.hbase.util.FSUtils; @@ -162,7 +162,7 @@ import com.google.protobuf.ByteString; * A lot of the meta information for an HRegion now lives inside other HRegions * or in the HBaseMaster, so only basic testing is possible. */ -@Category({VerySlowRegionServerTests.class, MediumTests.class}) +@Category({FlakeyTests.class, MediumTests.class}) @SuppressWarnings("deprecation") public class TestHRegion { // Do not spin up clusters in here. If you need to spin up a cluster, do it diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java index 6e4030b..1142d56 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TestMetaTableAccessor; import org.apache.hadoop.hbase.client.Consistency; @@ -45,6 +44,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.util.StringUtils; @@ -60,7 +60,7 @@ import com.google.protobuf.ServiceException; * Tests for region replicas. Sad that we cannot isolate these without bringing up a whole * cluster. See {@link TestRegionServerNoMaster}. */ -@Category({RegionServerTests.class, MediumTests.class}) +@Category({VerySlowRegionServerTests.class, MediumTests.class}) public class TestRegionReplicas { private static final Log LOG = LogFactory.getLog(TestRegionReplicas.class); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java index 1f6bd1c..bd17ede 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java @@ -73,7 +73,7 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -91,7 +91,7 @@ import org.mockito.Mockito; /** * Test replay of edits out of a WAL split. */ -@Category({RegionServerTests.class, MediumTests.class}) +@Category({VerySlowRegionServerTests.class, MediumTests.class}) public class TestWALReplay { public static final Log LOG = LogFactory.getLog(TestWALReplay.class); static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index 192009b..c5648ad 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -47,8 +48,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; + import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; @@ -62,7 +62,7 @@ import org.junit.experimental.categories.Category; /** * Test Export Snapshot Tool */ -@Category({VerySlowRegionServerTests.class, MediumTests.class}) +@Category({MapReduceTests.class, MediumTests.class}) public class TestExportSnapshot { private final Log LOG = LogFactory.getLog(getClass()); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java index 19d5965..ee72460 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java @@ -25,14 +25,14 @@ import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; +import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; /** * Reruns TestExportSnapshot using ExportSnapshot in secure mode. */ -@Category({VerySlowRegionServerTests.class, LargeTests.class}) +@Category({MapReduceTests.class, LargeTests.class}) public class TestSecureExportSnapshot extends TestExportSnapshot { @BeforeClass public static void setUpBeforeClass() throws Exception { diff --git pom.xml pom.xml index 1b97e38..40e8e7d 100644 --- pom.xml +++ pom.xml @@ -2099,7 +2099,7 @@ false - 2 + 1 1 false true @@ -2116,7 +2116,7 @@ false - 2 + 1 1 false true