From c7a59c9520ab961f29b5eb5ba15d1b22d0b87db9 Mon Sep 17 00:00:00 2001 From: stack Date: Thu, 29 Oct 2015 21:22:43 -0700 Subject: [PATCH] HBASE-14725 Vet categorization of tests so they for sure go into the right small/medium/large buckets Add category timeouts on a few tests that were missing timeouts (and that hung recently in tests) and then make it so all modules respect top-level test profiles runSmallTests, runMediumTests, and runLargeTests. Previous, only hbase-server did these profiles (which was confusing but made sense way back when when other modules were miniscule compared). Make it so we primitively scale up forked test count by CPU count. Previous was hard-coded at 5. Now make it a multiple of CPUs. Change category on a few tests that are medium by the time they take where they were marked as small (means they won't be running all up in a single JVM but will get their own JVM so less likely we see the timeout on first part that surefire throws from time to time. --- .../hadoop/hbase/client/TestAsyncProcess.java | 8 +++---- .../master/TestMetricsMasterSourceFactory.java | 6 ++--- .../procedure2/util/TestTimeoutBlockingQueue.java | 23 ++++++++---------- hbase-server/pom.xml | 9 ------- .../hadoop/hbase/ipc/TestRpcClientLeaks.java | 18 ++++++++------ .../hadoop/hbase/ipc/TestSimpleRpcScheduler.java | 5 ++++ .../hbase/master/cleaner/TestHFileLinkCleaner.java | 9 +++++-- .../hbase/regionserver/TestColumnSeeking.java | 9 ++++--- .../TestMultiVersionConcurrencyControl.java | 19 +++++++-------- .../regionserver/wal/TestLogRollingNoCluster.java | 9 +++++-- pom.xml | 28 ++++------------------ 11 files changed, 66 insertions(+), 77 deletions(-) diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java index b784f7a..22611ba 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java @@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -62,11 +63,13 @@ import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.rules.Timeout; +import org.junit.rules.TestRule; import org.mockito.Mockito; @Category({ClientTests.class, MediumTests.class}) public class TestAsyncProcess { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); private final static Log LOG = LogFactory.getLog(TestAsyncProcess.class); private static final TableName DUMMY_TABLE = TableName.valueOf("DUMMY_TABLE"); @@ -407,9 +410,6 @@ public class TestAsyncProcess { } } - @Rule - public Timeout timeout = new Timeout(10000); // 10 seconds max per method tested - @Test public void testSubmit() throws Exception { ClusterConnection hc = createHConnection(); diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java index 91efd39..3f97bb3 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/master/TestMetricsMasterSourceFactory.java @@ -19,9 +19,8 @@ package org.apache.hadoop.hbase.master; import org.apache.hadoop.hbase.CompatibilitySingletonFactory; -import org.apache.hadoop.hbase.master.MetricsMasterSource; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.MetricsTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -35,6 +34,5 @@ public class TestMetricsMasterSourceFactory { public void testGetInstanceNoHadoopCompat() throws Exception { //This should throw an exception because there is no compat lib on the class path. CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class); - } -} +} \ No newline at end of file diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java index aff536a..399cc97 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/util/TestTimeoutBlockingQueue.java @@ -19,28 +19,25 @@ package org.apache.hadoop.hbase.procedure2.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.util.Arrays; import java.util.concurrent.TimeUnit; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.TimeoutRetriever; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.MasterTests; - -import org.junit.Assert; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -@Category({MasterTests.class, SmallTests.class}) +@Category({MasterTests.class, MediumTests.class}) public class TestTimeoutBlockingQueue { - private static final Log LOG = LogFactory.getLog(TestTimeoutBlockingQueue.class); - + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); static class TestObject { private long timeout; private int seqId; diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 6ea1817..9b4d383 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -343,15 +343,6 @@ - - maven-surefire-plugin - ${surefire.version} - - - ${surefire.firstPartGroups} - - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java index 596b8ab..d3e5767 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestRpcClientLeaks.java @@ -17,15 +17,18 @@ */ package org.apache.hadoop.hbase.ipc; +import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.net.Socket; import java.net.SocketAddress; import java.util.List; -import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -36,7 +39,7 @@ import org.apache.hadoop.hbase.client.MetricsConnection; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.codec.Codec; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.io.compress.CompressionCodec; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -44,12 +47,14 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; +import org.junit.rules.TestRule; -import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1; -import static org.junit.Assert.*; +import com.google.common.collect.Lists; -@Category(SmallTests.class) +@Category(MediumTests.class) public class TestRpcClientLeaks { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); public static class MyRpcClientImpl extends RpcClientImpl { public static List savedSockets = Lists.newArrayList(); @@ -113,5 +118,4 @@ public class TestRpcClientLeaks { assertTrue("Socket + " + socket + " is not closed", socket.isClosed()); } } -} - +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java index 087429a..db992cd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestSimpleRpcScheduler.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.google.protobuf.Message; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Put; @@ -40,8 +41,10 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -65,6 +68,8 @@ import static org.mockito.Mockito.when; @Category({RPCTests.class, SmallTests.class}) public class TestSimpleRpcScheduler { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); private static final Log LOG = LogFactory.getLog(TestSimpleRpcScheduler.class); private final RpcScheduler.Context CONTEXT = new RpcScheduler.Context() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java index 66874e6..d6f1606 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -37,20 +38,24 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; /** * Test the HFileLink Cleaner. * HFiles with links cannot be deleted until a link is present. */ -@Category({MasterTests.class, SmallTests.class}) +@Category({MasterTests.class, MediumTests.class}) public class TestHFileLinkCleaner { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java index 2bd22a8..eca7703 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestColumnSeeking.java @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -43,18 +44,20 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.junit.rules.TestRule; -@Category({RegionServerTests.class, SmallTests.class}) +@Category({RegionServerTests.class, MediumTests.class}) public class TestColumnSeeking { @Rule public TestName name = new TestName(); - + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU(); private static final Log LOG = LogFactory.getLog(TestColumnSeeking.class); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java index a639e2c..1c2a4cd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiVersionConcurrencyControl.java @@ -17,21 +17,21 @@ */ package org.apache.hadoop.hbase.regionserver; -import junit.framework.TestCase; -import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.junit.experimental.categories.Category; - import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.junit.Assert; +import org.junit.experimental.categories.Category; + /** * This is a hammer test that verifies MultiVersionConcurrencyControl in a * multiple writer single reader scenario. */ -@Category({RegionServerTests.class, SmallTests.class}) -public class TestMultiVersionConcurrencyControl extends TestCase { +@Category({RegionServerTests.class, MediumTests.class}) +public class TestMultiVersionConcurrencyControl { static class Writer implements Runnable { final AtomicBoolean finished; final MultiVersionConcurrencyControl mvcc; @@ -47,7 +47,6 @@ public class TestMultiVersionConcurrencyControl extends TestCase { public boolean failed = false; public void run() { - AtomicLong startPoint = new AtomicLong(); while (!finished.get()) { MultiVersionConcurrencyControl.WriteEntry e = mvcc.begin(); @@ -125,9 +124,9 @@ public class TestMultiVersionConcurrencyControl extends TestCase { } // check failure. - assertFalse(readerFailed.get()); + Assert.assertFalse(readerFailed.get()); for (int i = 0; i < n; ++i) { - assertTrue(statuses[i].get()); + Assert.assertTrue(statuses[i].get()); } } } \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java index 08c1b15..0c68fc1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollingNoCluster.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; @@ -32,21 +33,25 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALKey; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TestRule; /** * Test many concurrent appenders to an WAL while rolling the log. */ -@Category({RegionServerTests.class, SmallTests.class}) +@Category({RegionServerTests.class, MediumTests.class}) public class TestLogRollingNoCluster { + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private final static byte [] EMPTY_1K_ARRAY = new byte[1024]; private static final int THREAD_COUNT = 100; // Spin up this many threads diff --git a/pom.xml b/pom.xml index 18eb9c4..0ab3d69 100644 --- a/pom.xml +++ b/pom.xml @@ -564,6 +564,7 @@ + ${surefire.firstPartGroups} false ${surefire.skipFirstPart} ${surefire.firstPartForkCount} @@ -1237,7 +1238,8 @@ false false 1 - 2 + + 1C org.apache.hadoop.hbase.testclassification.SmallTests org.apache.hadoop.hbase.testclassification.MediumTests false @@ -2282,27 +2284,7 @@ See as well the properties of the project for the values when no profile is active. --> - - nonParallelTests - - false - - - 1 - - - - - parallelTests - - false - - - 1 - - - - + singleJVMTests false @@ -2376,7 +2358,7 @@ 1 - 5 + 1C false false org.apache.hadoop.hbase.testclassification.SmallTests -- 2.6.1