From 3ff0a7424f7529c3c3348579c5f1ac4ddfc3aec2 Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Sun, 23 Jul 2017 12:57:23 -0500 Subject: [PATCH] HBASE-18440 ITs and Actions modify immutable TableDescriptors --- .../hbase/IntegrationTestIngestWithEncryption.java | 15 +++++++++----- .../hbase/chaos/actions/AddColumnAction.java | 21 +++++++++++-------- .../chaos/actions/ChangeSplitPolicyAction.java | 20 ++++++++++-------- .../chaos/actions/DecreaseMaxHFileSizeAction.java | 24 ++++++++++++++-------- .../hbase/mapreduce/IntegrationTestBulkLoad.java | 8 +++++--- 5 files changed, 55 insertions(+), 33 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java index 1f85a51ceb..1a7ca9124b 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestIngestWithEncryption.java @@ -24,6 +24,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileReaderImpl; @@ -94,14 +97,16 @@ public class IntegrationTestIngestWithEncryption extends IntegrationTestIngest { // Update the test table schema so HFiles from this point will be written with // encryption features enabled. final Admin admin = util.getAdmin(); - HTableDescriptor tableDescriptor = - new HTableDescriptor(admin.getTableDescriptor(getTablename())); - for (HColumnDescriptor columnDescriptor: tableDescriptor.getColumnFamilies()) { - columnDescriptor.setEncryptionType("AES"); + TableDescriptor tableDescriptor = admin.listTableDescriptor(getTablename()); + for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { + ColumnFamilyDescriptor updatedColumn = ColumnFamilyDescriptorBuilder + .newBuilder(columnDescriptor) + .setEncryptionType("AES") + .build(); LOG.info("Updating CF schema for " + getTablename() + "." + columnDescriptor.getNameAsString()); admin.disableTable(getTablename()); - admin.modifyColumnFamily(getTablename(), columnDescriptor); + admin.modifyColumnFamily(getTablename(), updatedColumn); admin.enableTable(getTablename()); util.waitFor(30000, 1000, true, new Predicate() { @Override diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java index f3827473a3..b21494be94 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java @@ -21,10 +21,12 @@ package org.apache.hadoop.hbase.chaos.actions; import java.io.IOException; import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; /** * Action the adds a column family to a table. @@ -45,12 +47,12 @@ public class AddColumnAction extends Action { @Override public void perform() throws Exception { - HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); - HColumnDescriptor columnDescriptor = null; + TableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); + ColumnFamilyDescriptor columnDescriptor = null; while(columnDescriptor == null || - tableDescriptor.getFamily(columnDescriptor.getName()) != null) { - columnDescriptor = new HColumnDescriptor(RandomStringUtils.randomAlphabetic(5)); + tableDescriptor.getColumnFamily(columnDescriptor.getName()) != null) { + columnDescriptor = ColumnFamilyDescriptorBuilder.of(RandomStringUtils.randomAlphabetic(5)); } // Don't try the modify if we're stopping @@ -59,8 +61,9 @@ public class AddColumnAction extends Action { } LOG.debug("Performing action: Adding " + columnDescriptor + " to " + tableName); - - tableDescriptor.addFamily(columnDescriptor); - admin.modifyTable(tableName, tableDescriptor); + TableDescriptor modifiedTable = TableDescriptorBuilder.newBuilder(tableDescriptor) + .addColumnFamily(columnDescriptor) + .build(); + admin.modifyTable(modifiedTable); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java index 9af572831a..b8a1afda15 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java @@ -17,20 +17,21 @@ */ package org.apache.hadoop.hbase.chaos.actions; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.IncreasingToUpperBoundRegionSplitPolicy; +import java.io.IOException; import java.util.Random; public class ChangeSplitPolicyAction extends Action { private final TableName tableName; private final String[] possiblePolicies; private final Random random; + private Admin admin; public ChangeSplitPolicyAction(TableName tableName) { this.tableName = tableName; @@ -42,17 +43,20 @@ public class ChangeSplitPolicyAction extends Action { this.random = new Random(); } + @Override + public void init(ActionContext context) throws IOException { + super.init(context); + this.admin = context.getHBaseIntegrationTestingUtility().getAdmin(); + } @Override public void perform() throws Exception { - HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); - Admin admin = util.getAdmin(); - LOG.info("Performing action: Change split policy of table " + tableName); - HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); String chosenPolicy = possiblePolicies[random.nextInt(possiblePolicies.length)]; - tableDescriptor.setRegionSplitPolicyClassName(chosenPolicy); + TableDescriptorBuilder builder = TableDescriptorBuilder + .newBuilder(admin.listTableDescriptor(tableName)) + .setRegionSplitPolicyClassName(chosenPolicy); LOG.info("Changing " + tableName + " split policy to " + chosenPolicy); - admin.modifyTable(tableName, tableDescriptor); + admin.modifyTable(builder.build()); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java index 98babeb651..96b4122627 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java @@ -18,12 +18,13 @@ package org.apache.hadoop.hbase.chaos.actions; -import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import java.io.IOException; import java.util.Random; public class DecreaseMaxHFileSizeAction extends Action { @@ -33,6 +34,7 @@ public class DecreaseMaxHFileSizeAction extends Action { private final long sleepTime; private final TableName tableName; private final Random random; + private Admin admin; public DecreaseMaxHFileSizeAction(long sleepTime, TableName tableName) { this.sleepTime = sleepTime; @@ -41,13 +43,17 @@ public class DecreaseMaxHFileSizeAction extends Action { } @Override + public void init(ActionContext context) throws IOException { + super.init(context); + this.admin = context.getHBaseIntegrationTestingUtility().getAdmin(); + } + + @Override public void perform() throws Exception { - HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); - Admin admin = util.getAdmin(); - HTableDescriptor htd = admin.getTableDescriptor(tableName); + TableDescriptor td = admin.listTableDescriptor(tableName); // Try and get the current value. - long currentValue = htd.getMaxFileSize(); + long currentValue = td.getMaxFileSize(); // If the current value is not set use the default for the cluster. // If configs are really weird this might not work. @@ -66,7 +72,9 @@ public class DecreaseMaxHFileSizeAction extends Action { newValue = Math.max(minFileSize, newValue) - (512 - random.nextInt(1024)); // Change the table descriptor. - htd.setMaxFileSize(newValue); + TableDescriptor modifiedTable = TableDescriptorBuilder.newBuilder(td) + .setMaxFileSize(newValue) + .build(); // Don't try the modify if we're stopping if (context.isStopping()) { @@ -74,7 +82,7 @@ public class DecreaseMaxHFileSizeAction extends Action { } // modify the table. - admin.modifyTable(tableName, htd); + admin.modifyTable(modifiedTable); // Sleep some time. if (sleepTime > 0) { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java index 52f1223e59..b5bbc8a712 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/mapreduce/IntegrationTestBulkLoad.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.mapreduce; import static org.junit.Assert.assertEquals; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.shaded.com.google.common.base.Joiner; import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets; @@ -33,7 +35,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestBase; import org.apache.hadoop.hbase.IntegrationTestingUtility; import org.apache.hadoop.hbase.KeyValue; @@ -202,8 +203,9 @@ public class IntegrationTestBulkLoad extends IntegrationTestBase { TableName t = getTablename(); Admin admin = util.getAdmin(); - HTableDescriptor desc = admin.getTableDescriptor(t); - desc.addCoprocessor(SlowMeCoproScanOperations.class.getName()); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(admin.listTableDescriptor(t)) + .addCoprocessor(SlowMeCoproScanOperations.class.getName()) + .build(); HBaseTestingUtility.modifyTableSync(admin, desc); } -- 2.13.3