Index: hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java (revision 1504302) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java (working copy) @@ -49,7 +49,7 @@ private static final String DEFAULT_TABLE_NAME = "cluster_test"; /** Column family used by the test */ - static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf"); + public static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf"); /** Column families used by the test */ static final byte[][] COLUMN_FAMILIES = { COLUMN_FAMILY }; Index: hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java =================================================================== --- hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java (revision 1504302) +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IngestIntegrationTestBase.java (working copy) @@ -33,7 +33,7 @@ * {@link LoadTestTool} to write and verify some data. */ public abstract class IngestIntegrationTestBase { - private static String tableName = null; + protected static String tableName = null; /** A soft limit on how long we should run */ private static final String RUN_TIME_KEY = "hbase.%s.runtime"; Index: hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestModifyColumns.java =================================================================== --- hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestModifyColumns.java (revision 0) +++ hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestModifyColumns.java (revision 0) @@ -0,0 +1,199 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ChaosMonkey; +import org.apache.hadoop.hbase.util.LoadTestTool; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Random; + +/** + * IT that will have the following monkeys while LoadTesting: + * - Add a column + * - Remove a column + * - Change the number of versions + * - Change the encoding + */ +@Category(IntegrationTests.class) +public class IntegrationTestModifyColumns extends IngestIntegrationTestBase { + + private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000; // run for 5 min by default + + /** How often to introduce the chaos. If too frequent, sequence of kills on minicluster + * can cause test to fail when Put runs out of retries. */ + private static final long CHAOS_EVERY_MS = 30 * 1000; + + private ChaosMonkey monkey; + + @Before + public void setUp() throws Exception { + Configuration conf = HBaseConfiguration.create(); + super.setUp(1, conf); + + ChaosMonkey.Policy chaosPolicy = new ChaosMonkey.PeriodicRandomActionPolicy( + CHAOS_EVERY_MS, + new AddColumnAction(tableName, new HBaseAdmin(util.getConfiguration())), + new RemoveColumnAction(tableName, new HBaseAdmin(util.getConfiguration())), + new ChangeVersionsAction(tableName, new HBaseAdmin(util.getConfiguration())), + new ChangeEncodingAction(tableName, new HBaseAdmin(util.getConfiguration())) + ); + monkey = new ChaosMonkey(util, chaosPolicy); + monkey.start(); + } + + private class AddColumnAction extends ChaosMonkey.Action { + + private byte[] tableName; + private HBaseAdmin admin; + + private AddColumnAction(String tableName, HBaseAdmin admin) { + this.admin = admin; + this.tableName = Bytes.toBytes(tableName); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor columnDescriptor = null; + + while(columnDescriptor == null || tableDescriptor.getFamily(columnDescriptor.getName()) != null) { + columnDescriptor = new HColumnDescriptor(RandomStringUtils.randomAlphabetic(5)); + } + + tableDescriptor.addFamily(columnDescriptor); + admin.modifyTable(tableName, tableDescriptor); + } + } + + private class RemoveColumnAction extends ChaosMonkey.Action { + private byte[] tableName; + private HBaseAdmin admin; + private Random random; + + private RemoveColumnAction(String tableName, HBaseAdmin admin) { + this.admin = admin; + this.tableName = Bytes.toBytes(tableName); + random = new Random(); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); + + if (columnDescriptors.length <= 1) { + return; + } + + int index = random.nextInt(columnDescriptors.length); + while(columnDescriptors[index].getNameAsString().equals(Bytes.toString(LoadTestTool.COLUMN_FAMILY))) { + index = random.nextInt(columnDescriptors.length); + } + + tableDescriptor.removeFamily(columnDescriptors[index].getName()); + + admin.modifyTable(tableName, tableDescriptor); + } + } + + private class ChangeVersionsAction extends ChaosMonkey.Action { + private byte[] tableName; + private HBaseAdmin admin; + private Random random; + + private ChangeVersionsAction(String tableName, HBaseAdmin admin) { + this.tableName = Bytes.toBytes(tableName); + this.admin = admin; + random = new Random(); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); + + if ( columnDescriptors == null || columnDescriptors.length == 0) { + return; + } + + int versions = random.nextInt(3) + 1; + for(HColumnDescriptor descriptor:columnDescriptors) { + descriptor.setMaxVersions(versions); + descriptor.setMinVersions(versions); + } + + admin.modifyTable(tableName, tableDescriptor); + } + } + + private class ChangeEncodingAction extends ChaosMonkey.Action { + private byte[] tableName; + private HBaseAdmin admin; + private Random random; + + private ChangeEncodingAction(String tableName, HBaseAdmin admin) { + this.tableName = Bytes.toBytes(tableName); + this.admin = admin; + random = new Random(); + } + + @Override + public void perform() throws Exception { + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); + + if (columnDescriptors == null || columnDescriptors.length == 0) { + return; + } + + // possible DataBlockEncoding id's + int[] possibleIds = {0, 2, 3, 4, 6}; + for (HColumnDescriptor descriptor : columnDescriptors) { + short id = (short) possibleIds[random.nextInt(possibleIds.length)]; + descriptor.setDataBlockEncoding(DataBlockEncoding.getEncodingById(id)); + } + + admin.modifyTable(tableName, tableDescriptor); + } + } + + @After + public void tearDown() throws Exception { + if (monkey != null) { + monkey.stop("tearDown"); + monkey.waitForStop(); + } + super.tearDown(); + } + + @Test + public void testDataIngest() throws Exception { + long time = DEFAULT_RUN_TIME; + runIngestTest(time, 2500, 10, 100, 20); + } +} Index: hbase-common/src/main/resources/hbase-default.xml =================================================================== --- hbase-common/src/main/resources/hbase-default.xml (revision 1504302) +++ hbase-common/src/main/resources/hbase-default.xml (working copy) @@ -837,7 +837,7 @@ hbase.online.schema.update.enable - false + true Set true to enable online schema changes. This is an experimental feature. There are known issues modifying table schemas at the same time a region split is happening so your table needs to be quiescent or else you have to