From f430cb00f2539e3c34d1a0857fe3cf26636884dd Mon Sep 17 00:00:00 2001 From: zhangduo Date: Sat, 14 Mar 2015 13:24:46 +0800 Subject: [PATCH] HBASE-13242 TestPerColumnFamilyFlush.testFlushingWhenLogRolling hung --- .../regionserver/TestPerColumnFamilyFlush.java | 67 +++++++++++++--------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java index 4a9b38a..07c3578 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -49,10 +50,10 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.Threads; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -61,7 +62,7 @@ import com.google.common.hash.Hashing; /** * This test verifies the correctness of the Per Column Family flushing strategy */ -@Category(LargeTests.class) +@Category({ RegionServerTests.class, LargeTests.class }) public class TestPerColumnFamilyFlush { private static final Log LOG = LogFactory.getLog(TestPerColumnFamilyFlush.class); @@ -119,7 +120,7 @@ public class TestPerColumnFamilyFlush { Arrays.equals(r.getFamilyMap(family).get(qf), val)); } - @Test (timeout=180000) + @Test(timeout = 180000) public void testSelectiveFlushWhenEnabled() throws IOException { // Set up the configuration Configuration conf = HBaseConfiguration.create(); @@ -258,7 +259,7 @@ public class TestPerColumnFamilyFlush { HBaseTestingUtility.closeRegionAndWAL(region); } - @Test (timeout=180000) + @Test(timeout = 180000) public void testSelectiveFlushWhenNotEnabled() throws IOException { // Set up the configuration Configuration conf = HBaseConfiguration.create(); @@ -337,7 +338,7 @@ public class TestPerColumnFamilyFlush { try { TEST_UTIL.startMiniCluster(numRegionServers); TEST_UTIL.getHBaseAdmin().createNamespace( - NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); + NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); HTable table = TEST_UTIL.createTable(TABLENAME, FAMILIES); HTableDescriptor htd = table.getTableDescriptor(); @@ -414,14 +415,14 @@ public class TestPerColumnFamilyFlush { // In distributed log replay, the log splitters ask the master for the // last flushed sequence id for a region. This test would ensure that we // are doing the book-keeping correctly. - @Test (timeout=180000) + @Test(timeout = 180000) public void testLogReplayWithDistributedReplay() throws Exception { TEST_UTIL.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); doTestLogReplay(); } // Test Log Replay with Distributed log split on. - @Test (timeout=180000) + @Test(timeout = 180000) public void testLogReplayWithDistributedLogSplit() throws Exception { TEST_UTIL.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); doTestLogReplay(); @@ -433,7 +434,7 @@ public class TestPerColumnFamilyFlush { * test ensures that we do a full-flush in that scenario. * @throws IOException */ - @Test (timeout=180000) + @Test(timeout = 180000) public void testFlushingWhenLogRolling() throws Exception { TableName tableName = TableName.valueOf("testFlushingWhenLogRolling"); Configuration conf = TEST_UTIL.getConfiguration(); @@ -447,9 +448,8 @@ public class TestPerColumnFamilyFlush { // Keep the block size small so that we fill up the log files very fast. conf.setLong("hbase.regionserver.hlog.blocksize", 6144); // Make it 10 as max logs before a flush comes on. - final int walcount = 10; - conf.setInt("hbase.regionserver.maxlogs", walcount); - int maxLogs = conf.getInt("hbase.regionserver.maxlogs", walcount); + final int maxLogs = 10; + conf.setInt("hbase.regionserver.maxlogs", maxLogs); final int numRegionServers = 4; try { @@ -462,7 +462,7 @@ public class TestPerColumnFamilyFlush { try (Admin admin = TEST_UTIL.getConnection().getAdmin()) { admin.flush(TableName.NAMESPACE_TABLE_NAME); } - HRegion desiredRegion = getRegionWithName(tableName).getFirst(); + final HRegion desiredRegion = getRegionWithName(tableName).getFirst(); assertTrue("Could not find a region which hosts the new region.", desiredRegion != null); LOG.info("Writing to region=" + desiredRegion); @@ -476,16 +476,32 @@ public class TestPerColumnFamilyFlush { table.flushCommits(); // Keep adding until we exceed the number of log files, so that we are // able to trigger the cleaning of old log files. - int currentNumLogFiles = ((FSHLog) (desiredRegion.getWAL())).getNumLogFiles(); - if (currentNumLogFiles > maxLogs) { - LOG.info("The number of log files is now: " + currentNumLogFiles + int currentNumRolledLogFiles = ((FSHLog) (desiredRegion.getWAL())).getNumRolledLogFiles(); + if (currentNumRolledLogFiles > maxLogs) { + LOG.info("The number of rolled log files is now: " + currentNumRolledLogFiles + ". Expect a log roll and memstore flush."); break; } } table.close(); // Wait for some time till the flush caused by log rolling happens. - while (((FSHLog) (desiredRegion.getWAL())).getNumLogFiles() > maxLogs) Threads.sleep(100); + TEST_UTIL.waitFor(30000, new Waiter.ExplainingPredicate() { + + @Override + public boolean evaluate() throws Exception { + return ((FSHLog) (desiredRegion.getWAL())).getNumRolledLogFiles() <= maxLogs; + } + + @Override + public String explainFailure() throws Exception { + int currentNumRolledLogFiles = ((FSHLog) (desiredRegion.getWAL())).getNumRolledLogFiles(); + if (currentNumRolledLogFiles > maxLogs) { + return "Still have " + currentNumRolledLogFiles + " rolled log files, expect below " + + maxLogs; + } + return "Unknown"; + } + }); LOG.info("Finished waiting on flush after too many WALs..."); // We have artificially created the conditions for a log roll. When a @@ -493,12 +509,9 @@ public class TestPerColumnFamilyFlush { // case here. // Individual families should have been flushed. - assertEquals(DefaultMemStore.DEEP_OVERHEAD, - desiredRegion.getStore(FAMILY1).getMemStoreSize()); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, - desiredRegion.getStore(FAMILY2).getMemStoreSize()); - assertEquals(DefaultMemStore.DEEP_OVERHEAD, - desiredRegion.getStore(FAMILY3).getMemStoreSize()); + assertEquals(DefaultMemStore.DEEP_OVERHEAD, desiredRegion.getStore(FAMILY1).getMemStoreSize()); + assertEquals(DefaultMemStore.DEEP_OVERHEAD, desiredRegion.getStore(FAMILY2).getMemStoreSize()); + assertEquals(DefaultMemStore.DEEP_OVERHEAD, desiredRegion.getStore(FAMILY3).getMemStoreSize()); // And of course, the total memstore should also be clean. assertEquals(0, desiredRegion.getMemstoreSize().get()); @@ -534,7 +547,7 @@ public class TestPerColumnFamilyFlush { // Under the same write load, small stores should have less store files when // percolumnfamilyflush enabled. - @Test (timeout=180000) + @Test(timeout = 180000) public void testCompareStoreFileCount() throws Exception { long memstoreFlushSize = 1024L * 1024; Configuration conf = TEST_UTIL.getConfiguration(); @@ -561,7 +574,7 @@ public class TestPerColumnFamilyFlush { try { TEST_UTIL.startMiniCluster(1); TEST_UTIL.getHBaseAdmin().createNamespace( - NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); + NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); TEST_UTIL.getHBaseAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(TABLENAME); Connection conn = ConnectionFactory.createConnection(conf); @@ -578,12 +591,12 @@ public class TestPerColumnFamilyFlush { TEST_UTIL.shutdownMiniCluster(); } - LOG.info("==============Test with selective flush enabled==============="); - conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName()); + LOG.info("==============Test with selective flush enabled==============="); + conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName()); try { TEST_UTIL.startMiniCluster(1); TEST_UTIL.getHBaseAdmin().createNamespace( - NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); + NamespaceDescriptor.create(TABLENAME.getNamespaceAsString()).build()); TEST_UTIL.getHBaseAdmin().createTable(htd); Connection conn = ConnectionFactory.createConnection(conf); Table table = conn.getTable(TABLENAME); -- 1.9.1