From 31320d55ed6da382f3641e6f90c4effa6f2ff8cc Mon Sep 17 00:00:00 2001 From: eshcar Date: Wed, 1 Feb 2017 15:06:19 +0200 Subject: [PATCH] HBASE-17575: Run critical tests with each of the Inmemory Compaction Policies enabled --- .../hbase/IntegrationTestAcidGuarantees.java | 4 +- .../regionserver/MemStoreSegmentsIterator.java | 8 +- .../apache/hadoop/hbase/TestAcidGuarantees.java | 90 ++++++++++------------ .../org/apache/hadoop/hbase/TestIOFencing.java | 19 +++-- .../hadoop/hbase/backup/TestHFileArchiving.java | 3 - .../hbase/client/TestMobSnapshotFromClient.java | 5 -- .../client/TestSnapshotCloneIndependence.java | 11 ++- .../hbase/client/TestSnapshotFromClient.java | 9 ++- .../TestMasterProcedureSchedulerConcurrency.java | 5 -- .../hadoop/hbase/regionserver/TestCompaction.java | 10 ++- .../hbase/regionserver/TestMajorCompaction.java | 26 +++++-- .../regionserver/TestPerColumnFamilyFlush.java | 19 ++--- .../hbase/regionserver/TestRecoveredEdits.java | 33 ++++++-- .../regionserver/wal/AbstractTestLogRolling.java | 13 ++-- .../regionserver/wal/TestAsyncLogRolling.java | 2 - .../snapshot/TestFlushSnapshotFromClient.java | 10 ++- 16 files changed, 150 insertions(+), 117 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java index 003a51f..ca8e277 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestAcidGuarantees.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hbase; import com.google.common.collect.Sets; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.MemStoreCompactor; import org.apache.hadoop.hbase.testclassification.IntegrationTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.ToolRunner; @@ -74,7 +76,7 @@ public class IntegrationTestAcidGuarantees extends IntegrationTestBase { // replace the HBaseTestingUtility in the unit test with the integration test's // IntegrationTestingUtility - tag = new TestAcidGuarantees(); + tag = new TestAcidGuarantees(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT); tag.setHBaseTestingUtil(util); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSegmentsIterator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSegmentsIterator.java index 8790bc2..e2f4ebb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSegmentsIterator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSegmentsIterator.java @@ -51,10 +51,10 @@ public abstract class MemStoreSegmentsIterator implements Iterator { // list of Scanners of segments in the pipeline, when compaction starts List scanners = new ArrayList(); - // create the list of scanners with the smallest read point, meaning that - // only relevant KVs are going to be returned by the pipeline traversing - for (Segment segment : segments) { - scanners.add(segment.getScanner(store.getSmallestReadPoint())); + // create the list of scanners to traverse over all the data + // no dirty reads here as these are immutable segments + for (ImmutableSegment segment : segments) { + scanners.add(segment.getScanner(Integer.MAX_VALUE)); } scanner = new MemStoreScanner(comparator, scanners, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java index 59ca64c..569ca89 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuarantees.java @@ -39,16 +39,21 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.MemStoreLAB; import org.apache.hadoop.hbase.testclassification.FlakeyTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import com.google.common.collect.Lists; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; /** * Test case that uses multiple threads to read and write multifamily rows @@ -58,7 +63,12 @@ import com.google.common.collect.Lists; * a real cluster (eg for testing with failures, region movement, etc) */ @Category({FlakeyTests.class, MediumTests.class}) +@RunWith(Parameterized.class) public class TestAcidGuarantees implements Tool { + @Parameterized.Parameters + public static Object[] data() { + return new Object[] { "NONE", "BASIC", "EAGER" }; + } protected static final Log LOG = LogFactory.getLog(TestAcidGuarantees.class); public static final TableName TABLE_NAME = TableName.valueOf("TestAcidGuarantees"); public static final byte [] FAMILY_A = Bytes.toBytes("A"); @@ -93,16 +103,19 @@ public class TestAcidGuarantees implements Tool { } } - public TestAcidGuarantees() { + public TestAcidGuarantees(String compType) { // Set small flush size for minicluster so we exercise reseeking scanners Configuration conf = HBaseConfiguration.create(); - conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128*1024)); + conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024)); // prevent aggressive region split conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, - ConstantSizeRegionSplitPolicy.class.getName()); + ConstantSizeRegionSplitPolicy.class.getName()); conf.setInt("hfile.format.version", 3); // for mob tests - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); + conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, compType); + if(MemoryCompactionPolicy.valueOf(compType) == MemoryCompactionPolicy.EAGER) { + conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false); + conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.9); + } util = new HBaseTestingUtility(conf); } @@ -390,70 +403,50 @@ public class TestAcidGuarantees implements Tool { } } + @Before + public void setUp() throws Exception { + util.startMiniCluster(1); + } + + @After + public void tearDown() throws Exception { + util.shutdownMiniCluster(); + } + @Test public void testGetAtomicity() throws Exception { - util.startMiniCluster(1); - try { - runTestAtomicity(20000, 5, 5, 0, 3); - } finally { - util.shutdownMiniCluster(); - } + runTestAtomicity(20000, 5, 5, 0, 3); } @Test public void testScanAtomicity() throws Exception { - util.startMiniCluster(1); - try { - runTestAtomicity(20000, 5, 0, 5, 3); - } finally { - util.shutdownMiniCluster(); - } + runTestAtomicity(20000, 5, 0, 5, 3); } @Test public void testMixedAtomicity() throws Exception { - util.startMiniCluster(1); - try { - runTestAtomicity(20000, 5, 2, 2, 3); - } finally { - util.shutdownMiniCluster(); - } + runTestAtomicity(20000, 5, 2, 2, 3); } @Test public void testMobGetAtomicity() throws Exception { - util.startMiniCluster(1); - try { - boolean systemTest = false; - boolean useMob = true; - runTestAtomicity(20000, 5, 5, 0, 3, systemTest, useMob); - } finally { - util.shutdownMiniCluster(); - } + boolean systemTest = false; + boolean useMob = true; + runTestAtomicity(20000, 5, 5, 0, 3, systemTest, useMob); } @Test public void testMobScanAtomicity() throws Exception { - util.startMiniCluster(1); - try { - boolean systemTest = false; - boolean useMob = true; - runTestAtomicity(20000, 5, 0, 5, 3, systemTest, useMob); - } finally { - util.shutdownMiniCluster(); - } + boolean systemTest = false; + boolean useMob = true; + runTestAtomicity(20000, 5, 0, 5, 3, systemTest, useMob); } @Test public void testMobMixedAtomicity() throws Exception { - util.startMiniCluster(1); - try { - boolean systemTest = false; - boolean useMob = true; - runTestAtomicity(20000, 5, 2, 2, 3, systemTest, useMob); - } finally { - util.shutdownMiniCluster(); - } + boolean systemTest = false; + boolean useMob = true; + runTestAtomicity(20000, 5, 2, 2, 3, systemTest, useMob); } //////////////////////////////////////////////////////////////////////////// @@ -488,7 +481,8 @@ public class TestAcidGuarantees implements Tool { Configuration c = HBaseConfiguration.create(); int status; try { - TestAcidGuarantees test = new TestAcidGuarantees(); + TestAcidGuarantees test = new TestAcidGuarantees(CompactingMemStore + .COMPACTING_MEMSTORE_TYPE_DEFAULT); status = ToolRunner.run(c, test, args); } catch (Exception e) { LOG.error("Exiting due to error", e); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java index eb2fa92..f12a334 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestIOFencing.java @@ -223,7 +223,9 @@ public class TestIOFencing { */ @Test public void testFencingAroundCompaction() throws Exception { - doTest(BlockCompactionsInPrepRegion.class); + for(MemoryCompactionPolicy policy : MemoryCompactionPolicy.values()) { + doTest(BlockCompactionsInPrepRegion.class, policy); + } } /** @@ -234,10 +236,12 @@ public class TestIOFencing { */ @Test public void testFencingAroundCompactionAfterWALSync() throws Exception { - doTest(BlockCompactionsInCompletionRegion.class); + for(MemoryCompactionPolicy policy : MemoryCompactionPolicy.values()) { + doTest(BlockCompactionsInCompletionRegion.class, policy); + } } - public void doTest(Class regionClass) throws Exception { + public void doTest(Class regionClass, MemoryCompactionPolicy policy) throws Exception { Configuration c = TEST_UTIL.getConfiguration(); // Insert our custom region c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class); @@ -250,8 +254,7 @@ public class TestIOFencing { c.setLong("hbase.hstore.blockingStoreFiles", 1000); // Compact quickly after we tell it to! c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000); - c.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); + c.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy)); LOG.info("Starting mini cluster"); TEST_UTIL.startMiniCluster(1); CompactionBlockerRegion compactingRegion = null; @@ -344,7 +347,11 @@ public class TestIOFencing { Thread.sleep(1000); assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000); } - assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table)); + if(policy == MemoryCompactionPolicy.EAGER) { + assertTrue(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT >= TEST_UTIL.countRows(table)); + } else { + assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table)); + } } finally { if (compactingRegion != null) { compactingRegion.allowCompactions(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index 8b5abb3..2e3dc46 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -89,9 +89,6 @@ public class TestHFileArchiving { // prevent aggressive region split conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); - // no memory compaction - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); } @After diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java index 05790c3..268bc14 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMobSnapshotFromClient.java @@ -20,10 +20,7 @@ package org.apache.hadoop.hbase.client; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -53,8 +50,6 @@ public class TestMobSnapshotFromClient extends TestSnapshotFromClient { protected static void setupConf(Configuration conf) { TestSnapshotFromClient.setupConf(conf); conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java index 04bc6c7..66c5abf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotCloneIndependence.java @@ -24,14 +24,19 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.junit.After; @@ -112,8 +117,6 @@ public class TestSnapshotCloneIndependence { // will even trigger races between creating the directory containing back references and // the back reference itself. conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); } @Before diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java index 633c708..c813faf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java @@ -30,9 +30,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; @@ -93,8 +96,6 @@ public class TestSnapshotFromClient { conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java index b530640..df431a5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureSchedulerConcurrency.java @@ -27,12 +27,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.MemoryCompactionPolicy; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.procedure.TestMasterProcedureScheduler.TestTableProcedure; import org.apache.hadoop.hbase.procedure2.Procedure; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -54,8 +51,6 @@ public class TestMasterProcedureSchedulerConcurrency { @Before public void setUp() throws IOException { conf = HBaseConfiguration.create(); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); queue = new MasterProcedureScheduler(conf); queue.start(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index f254792..bc51c41 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -43,7 +43,13 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.ChoreService; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; @@ -102,8 +108,6 @@ public class TestCompaction { conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100); conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY, NoLimitThroughputController.class.getName()); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3); secondRowBytes = START_KEY_BYTES.clone(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java index 19ed232..edc46ca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java @@ -37,7 +37,12 @@ import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; @@ -60,14 +65,20 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; - +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; /** * Test major compactions */ @Category({RegionServerTests.class, MediumTests.class}) +@RunWith(Parameterized.class) public class TestMajorCompaction { - @Rule public TestName name = new TestName(); + @Parameterized.Parameters + public static Object[] data() { + return new Object[] { "NONE", "BASIC", "EAGER" }; + } + @Rule public TestName name; private static final Log LOG = LogFactory.getLog(TestMajorCompaction.class.getName()); private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU(); protected Configuration conf = UTIL.getConfiguration(); @@ -82,15 +93,14 @@ public class TestMajorCompaction { private static final long MAX_FILES_TO_COMPACT = 10; /** constructor */ - public TestMajorCompaction() { + public TestMajorCompaction(String compType) { super(); - + name = new TestName(); // Set cache flush size to 1MB conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024); conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100); compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); + conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compType)); secondRowBytes = START_KEY_BYTES.clone(); // Increment the least significant character so we get to next row. @@ -101,7 +111,7 @@ public class TestMajorCompaction { @Before public void setUp() throws Exception { - this.htd = UTIL.createTableDescriptor(name.getMethodName()); + this.htd = UTIL.createTableDescriptor(name.getMethodName().replace('[','i').replace(']','i')); this.r = UTIL.createLocalHRegion(htd, null, null); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java index dc87d2f..de01d4a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java @@ -22,7 +22,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -74,12 +83,6 @@ public class TestPerColumnFamilyFlush { public static final byte[] FAMILY3 = FAMILIES[2]; - @Before - public void setUp() throws IOException { - TEST_UTIL.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); - } - private HRegion initHRegion(String callingMethod, Configuration conf) throws IOException { HTableDescriptor htd = new HTableDescriptor(TABLENAME); for (byte[] family : FAMILIES) { @@ -127,8 +130,6 @@ public class TestPerColumnFamilyFlush { conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushAllLargeStoresPolicy.class.getName()); conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN, 40 * 1024); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); // Intialize the region Region region = initHRegion("testSelectiveFlushWithDataCompaction", conf); // Add 1200 entries for CF1, 100 for CF2 and 50 for CF3 diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java index 553075f..4315bd4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRecoveredEdits.java @@ -28,7 +28,17 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellScanner; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MemoryCompactionPolicy; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; @@ -60,13 +70,20 @@ public class TestRecoveredEdits { * made it in. * @throws IOException */ - @Test (timeout=60000) - public void testReplayWorksThoughLotsOfFlushing() throws IOException { + @Test (timeout=180000) + public void testReplayWorksThoughLotsOfFlushing() throws + IOException { + for(MemoryCompactionPolicy policy : MemoryCompactionPolicy.values()) { + testReplayWorksWithMemoryCompactionPolicy(policy); + } + } + + private void testReplayWorksWithMemoryCompactionPolicy(MemoryCompactionPolicy policy) throws + IOException { Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); // Set it so we flush every 1M or so. Thats a lot. conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); + conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy)); // The file of recovered edits has a column family of 'meta'. Also has an encoded regionname // of 4823016d8fca70b25503ee07f4c6d79f which needs to match on replay. final String encodedRegionName = "4823016d8fca70b25503ee07f4c6d79f"; @@ -122,7 +139,11 @@ public class TestRecoveredEdits { // Our 0000000000000016310 is 10MB. Most of the edits are for one region. Lets assume that if // we flush at 1MB, that there are at least 3 flushed files that are there because of the // replay of edits. - assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10); + if(policy == MemoryCompactionPolicy.EAGER) { + assertTrue("Files count=" + storeFiles.size(), storeFiles.size() >= 1); + } else { + assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10); + } // Now verify all edits made it into the region. int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region); LOG.info("Checked " + count + " edits made it in"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java index c3fafb6..a11540c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java @@ -27,13 +27,19 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Store; @@ -107,9 +113,6 @@ public abstract class AbstractTestLogRolling { // Reduce thread wake frequency so that other threads can get // a chance to run. conf.setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000); - - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); } @Before diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java index a412d27..fabf6d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java @@ -22,10 +22,8 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests; import org.apache.hadoop.hbase.wal.AsyncFSWALProvider; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 8509c9d..deb3320 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -34,13 +34,17 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.SnapshotType; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.regionserver.CompactingMemStore; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.client.SnapshotDescription; @@ -100,8 +104,6 @@ public class TestFlushSnapshotFromClient { conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName()); - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(MemoryCompactionPolicy.NONE)); } @Before -- 2.10.1 (Apple Git-78)