From 2eb00711422d5b3644c39a7e68ecac3eace58fdc Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Thu, 8 Aug 2013 15:40:57 -0700 Subject: [PATCH] Create an Integration Test for online schema change --- ...IntegrationTestDataIngestSlowDeterministic.java | 6 +- .../IntegrationTestDataIngestWithChaosMonkey.java | 18 +-- ...ntegrationTestBigLinkedListWithChaosMonkey.java | 18 +-- .../org/apache/hadoop/hbase/util/ChaosMonkey.java | 137 ++++++++++++++++++++ 4 files changed, 154 insertions(+), 25 deletions(-) diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java index edab23b..a3ab48c 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestSlowDeterministic.java @@ -76,7 +76,11 @@ public class IntegrationTestDataIngestSlowDeterministic extends IngestIntegratio Action[] actions2 = new Action[] { new SplitRandomRegionOfTable(tableName), new MergeRandomAdjacentRegionsOfTable(tableName), - new SnapshotTable(tableName) + new SnapshotTable(tableName), + new ChaosMonkey.AddColumnAction(tableName), + new ChaosMonkey.RemoveColumnAction(tableName), + new ChaosMonkey.ChangeEncodingAction(tableName), + new ChaosMonkey.ChangeVersionsAction(tableName) }; // Destructive actions to mess things around. diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java index 44c0616..4e585cc 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestDataIngestWithChaosMonkey.java @@ -20,17 +20,7 @@ package org.apache.hadoop.hbase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.ChaosMonkey; -import org.apache.hadoop.hbase.util.ChaosMonkey.Action; -import org.apache.hadoop.hbase.util.ChaosMonkey.CompactRandomRegionOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.CompactTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.FlushRandomRegionOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.FlushTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.MergeRandomAdjacentRegionsOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.MoveRandomRegionOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.MoveRegionsOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.PeriodicRandomActionPolicy; -import org.apache.hadoop.hbase.util.ChaosMonkey.SnapshotTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.SplitRandomRegionOfTable; +import org.apache.hadoop.hbase.util.ChaosMonkey.*; import org.apache.hadoop.hbase.util.LoadTestTool; import org.junit.After; import org.junit.Before; @@ -88,7 +78,11 @@ public class IntegrationTestDataIngestWithChaosMonkey extends IngestIntegrationT new SplitRandomRegionOfTable(tableName), new MergeRandomAdjacentRegionsOfTable(tableName), new SnapshotTable(tableName), - new MoveRegionsOfTable(tableName) + new MoveRegionsOfTable(tableName), + new AddColumnAction(tableName), + new RemoveColumnAction(tableName), + new ChangeEncodingAction(tableName), + new ChangeVersionsAction(tableName) }; monkey = new ChaosMonkey(util, diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java index 04ab382..bb7cdb4 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java @@ -25,17 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.IntegrationTests; import org.apache.hadoop.hbase.util.ChaosMonkey; -import org.apache.hadoop.hbase.util.ChaosMonkey.Action; -import org.apache.hadoop.hbase.util.ChaosMonkey.CompactRandomRegionOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.CompactTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.FlushRandomRegionOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.FlushTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.MergeRandomAdjacentRegionsOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.MoveRandomRegionOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.MoveRegionsOfTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.PeriodicRandomActionPolicy; -import org.apache.hadoop.hbase.util.ChaosMonkey.SnapshotTable; -import org.apache.hadoop.hbase.util.ChaosMonkey.SplitRandomRegionOfTable; +import org.apache.hadoop.hbase.util.ChaosMonkey.*; import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.Before; @@ -93,7 +83,11 @@ public class IntegrationTestBigLinkedListWithChaosMonkey extends IntegrationTest new SplitRandomRegionOfTable(tableName), new MergeRandomAdjacentRegionsOfTable(tableName), new SnapshotTable(tableName), - new MoveRegionsOfTable(tableName) + new MoveRegionsOfTable(tableName), + new AddColumnAction(tableName), + new RemoveColumnAction(tableName), + new ChangeEncodingAction(tableName), + new ChangeVersionsAction(tableName) }; monkey = new ChaosMonkey(util, diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java index 2140189..cafceb7 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/util/ChaosMonkey.java @@ -26,8 +26,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.Random; import org.apache.commons.cli.CommandLine; +import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,7 +38,9 @@ import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseCluster; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestDataIngestWithChaosMonkey; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.ServerLoad; @@ -44,6 +48,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; @@ -761,6 +766,138 @@ public class ChaosMonkey extends AbstractHBaseTool implements Stoppable { } } + public static class AddColumnAction extends ChaosMonkey.Action { + + private byte[] tableName; + private HBaseAdmin admin; + + public AddColumnAction(String tableName) { + this.tableName = Bytes.toBytes(tableName); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor columnDescriptor = null; + + while(columnDescriptor == null || + tableDescriptor.getFamily(columnDescriptor.getName()) != null) { + columnDescriptor = new HColumnDescriptor(RandomStringUtils.randomAlphabetic(5)); + } + + tableDescriptor.addFamily(columnDescriptor); + admin.modifyTable(tableName, tableDescriptor); + } + } + + public static class RemoveColumnAction extends ChaosMonkey.Action { + private byte[] tableName; + private HBaseAdmin admin; + private Random random; + + public RemoveColumnAction(String tableName) { + this.tableName = Bytes.toBytes(tableName); + random = new Random(); + } + + @Override + public void init(ActionContext context) throws IOException { + super.init(context); + this.admin = context.getHaseIntegrationTestingUtility().getHBaseAdmin(); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); + + if (columnDescriptors.length <= 1) { + return; + } + + int index = random.nextInt(columnDescriptors.length); + while(columnDescriptors[index].getNameAsString().equals( + Bytes.toString(LoadTestTool.COLUMN_FAMILY))) { + index = random.nextInt(columnDescriptors.length); + } + + tableDescriptor.removeFamily(columnDescriptors[index].getName()); + + admin.modifyTable(tableName, tableDescriptor); + } + } + + public static class ChangeVersionsAction extends ChaosMonkey.Action { + private byte[] tableName; + private HBaseAdmin admin; + private Random random; + + public ChangeVersionsAction(String tableName) { + this.tableName = Bytes.toBytes(tableName); + random = new Random(); + } + + @Override + public void init(ActionContext context) throws IOException { + super.init(context); + this.admin = context.getHaseIntegrationTestingUtility().getHBaseAdmin(); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); + + if ( columnDescriptors == null || columnDescriptors.length == 0) { + return; + } + + int versions = random.nextInt(3) + 1; + for(HColumnDescriptor descriptor:columnDescriptors) { + descriptor.setMaxVersions(versions); + descriptor.setMinVersions(versions); + } + + admin.modifyTable(tableName, tableDescriptor); + } + } + + public static class ChangeEncodingAction extends ChaosMonkey.Action { + private byte[] tableName; + private HBaseAdmin admin; + private Random random; + + public ChangeEncodingAction(String tableName) { + this.tableName = Bytes.toBytes(tableName); + random = new Random(); + } + + @Override + public void init(ActionContext context) throws IOException { + super.init(context); + this.admin = context.getHaseIntegrationTestingUtility().getHBaseAdmin(); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); + + if (columnDescriptors == null || columnDescriptors.length == 0) { + return; + } + + // possible DataBlockEncoding id's + int[] possibleIds = {0, 2, 3, 4, 6}; + for (HColumnDescriptor descriptor : columnDescriptors) { + short id = (short) possibleIds[random.nextInt(possibleIds.length)]; + descriptor.setDataBlockEncoding(DataBlockEncoding.getEncodingById(id)); + } + + admin.modifyTable(tableName, tableDescriptor); + } + } + /** * A context for a Policy */ -- 1.7.10.2 (Apple Git-33)