diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 1bd4e07..750d39e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -40,6 +40,8 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilySchema; @@ -185,7 +187,8 @@ public class HTableDescriptor implements Comparable { /** * INTERNAL Used by shell/rest interface to access this metadata - * attribute which denotes if the table should be treated by region normalizer. + * attribute which denotes the allowed types of action (split/merge) when the table is treated + * by region normalizer. * * @see #isNormalizationEnabled() */ @@ -220,9 +223,11 @@ public class HTableDescriptor implements Comparable { public static final boolean DEFAULT_COMPACTION_ENABLED = true; /** - * Constant that denotes whether the table is normalized by default. + * Constant that denotes the types of action the table is normalized by default where both split + * and merge are allowed. */ - public static final boolean DEFAULT_NORMALIZATION_ENABLED = false; + public static final Bytes DESIRED_NORMALIZATION_TYPES = + new Bytes(Bytes.toBytes("MS")); /** * Constant that denotes the maximum default size of the memstore after which @@ -249,7 +254,7 @@ public class HTableDescriptor implements Comparable { String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH)); DEFAULT_VALUES.put(DURABILITY, DEFAULT_DURABLITY.name()); //use the enum name DEFAULT_VALUES.put(REGION_REPLICATION, String.valueOf(DEFAULT_REGION_REPLICATION)); - DEFAULT_VALUES.put(NORMALIZATION_ENABLED, String.valueOf(DEFAULT_NORMALIZATION_ENABLED)); + DEFAULT_VALUES.put(NORMALIZATION_ENABLED, ""); for (String s : DEFAULT_VALUES.keySet()) { RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(s))); } @@ -640,22 +645,44 @@ public class HTableDescriptor implements Comparable { } /** - * Check if normalization enable flag of the table is true. If flag is - * false then no region normalizer won't attempt to normalize this table. + * Check if normalization flag of the table. If flag is + * empty then region normalizer won't attempt to normalize this table. * - * @return true if region normalization is enabled for this table + * @return List of PlanType if region normalization is enabled for this table + * null means region normalization is disabled */ - public boolean isNormalizationEnabled() { - return isSomething(NORMALIZATION_ENABLED_KEY, DEFAULT_NORMALIZATION_ENABLED); + public List getDesiredNormalizationTypes() { + byte [] value = getValue(NORMALIZATION_ENABLED_KEY); + if (value == null) { + return null; + } + String strValue = Bytes.toString(value); + if (strValue.isEmpty()) { + return null; + } + List types = new ArrayList<>(); + if (strValue.toUpperCase().contains("M")) { + types.add(PlanType.MERGE); + } + if (strValue.toUpperCase().contains("S")) { + types.add(PlanType.SPLIT); + } + return types; } /** * Setting the table normalization enable flag. * - * @param isEnable True if enable normalization. + * @param isEnable True if enable both types of region normalization. */ public HTableDescriptor setNormalizationEnabled(final boolean isEnable) { - setValue(NORMALIZATION_ENABLED_KEY, isEnable ? TRUE : FALSE); + setValue(NORMALIZATION_ENABLED_KEY, isEnable ? DESIRED_NORMALIZATION_TYPES : null); + return this; + } + + public HTableDescriptor setNormalizationEnabled(final String types) { + setValue(NORMALIZATION_ENABLED_KEY, types == null || types.isEmpty() ? null : + new Bytes(Bytes.toBytes(types.toUpperCase()))); return this; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/normalizer/NormalizationPlan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/normalizer/NormalizationPlan.java index 9f866d3..66481e6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/normalizer/NormalizationPlan.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/normalizer/NormalizationPlan.java @@ -16,7 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.master.normalizer; +package org.apache.hadoop.hbase.normalizer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 4feb2e7..c319bb1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -72,7 +72,7 @@ import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.quotas.QuotaExceededException; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 2e42acb..c0c2f97 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -92,7 +92,6 @@ import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.LogCleaner; import org.apache.hadoop.hbase.master.handler.DispatchMergingRegionHandler; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizer; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerChore; import org.apache.hadoop.hbase.master.normalizer.RegionNormalizerFactory; @@ -114,6 +113,8 @@ import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost; import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; @@ -1323,14 +1324,21 @@ public class HMaster extends HRegionServer implements MasterServices { for (TableName table : allEnabledTables) { TableDescriptor tblDesc = getTableDescriptors().getDescriptor(table); - if (table.isSystemTable() || (tblDesc != null && - tblDesc.getHTableDescriptor() != null && - !tblDesc.getHTableDescriptor().isNormalizationEnabled())) { - LOG.debug("Skipping normalization for table: " + table + ", as it's either system" - + " table or doesn't have auto normalization turned on"); - continue; + if (table.isSystemTable()) { + LOG.debug("Skipping normalization for table: " + table + ", as it's system table"); + continue; } - NormalizationPlan plan = this.normalizer.computePlanForTable(table); + List types = null; + if (tblDesc != null && + tblDesc.getHTableDescriptor() != null) { + types = tblDesc.getHTableDescriptor().getDesiredNormalizationTypes(); + if (types == null) { + LOG.debug("Skipping normalization for table: " + table + ", as it" + + " doesn't have auto normalization turned on"); + continue; + } + } + NormalizationPlan plan = this.normalizer.computePlanForTable(table, types); plan.execute(clusterConnection.getAdmin()); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java index 5aecc48..29cc0c3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/EmptyNormalizationPlan.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.master.normalizer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; /** * Plan which signifies that no normalization is required, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java index e2035bb..f3ce1d5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/MergeNormalizationPlan.java @@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; import java.io.IOException; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java index d60474d..616098e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java @@ -18,12 +18,15 @@ */ package org.apache.hadoop.hbase.master.normalizer; +import java.util.List; + import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; /** * Performs "normalization" of regions on the cluster, making sure that suboptimal @@ -47,9 +50,11 @@ public interface RegionNormalizer { /** * Computes next optimal normalization plan. * @param table table to normalize + * @param types desired types of NormalizationPlan * @return Next (perhaps most urgent) normalization action to perform */ - NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException; + NormalizationPlan computePlanForTable(TableName table, List types) + throws HBaseIOException; /** * Notification for the case where plan couldn't be executed due to constraint violation, such as diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index fe10bd1..1601c24 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -27,7 +27,8 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.util.Triple; import java.util.ArrayList; @@ -60,7 +61,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer { private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class); private static final int MIN_REGION_COUNT = 3; private MasterServices masterServices; - private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; + private static long[] skippedCount = new long[PlanType.values().length]; /** * Set the master service. @@ -102,10 +103,12 @@ public class SimpleRegionNormalizer implements RegionNormalizer { * Action may be either a split, or a merge, or no action. * * @param table table to normalize + * @param types desired types of NormalizationPlan * @return normalization plan to execute */ @Override - public NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException { + public NormalizationPlan computePlanForTable(TableName table, List types) + throws HBaseIOException { if (table == null || table.isSystemTable()) { LOG.debug("Normalization of system table " + table + " isn't allowed"); return EmptyNormalizationPlan.getInstance(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java index b95bfb7..76b7cc2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SplitNormalizationPlan.java @@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; import java.io.IOException; import java.util.Arrays; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index 970af43..c6b893f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -26,6 +26,8 @@ import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -53,6 +55,12 @@ public class TestSimpleRegionNormalizer { private static final Log LOG = LogFactory.getLog(TestSimpleRegionNormalizer.class); private static RegionNormalizer normalizer; + private static List bothTypes; + static { + bothTypes = new ArrayList<>(); + bothTypes.add(PlanType.SPLIT); + bothTypes.add(PlanType.MERGE); + } // mocks private static MasterServices masterServices; @@ -69,7 +77,7 @@ public class TestSimpleRegionNormalizer { Map regionSizes = new HashMap<>(); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue(plan instanceof EmptyNormalizationPlan); } @@ -88,7 +96,7 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri2.getRegionName(), 15); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue((plan instanceof EmptyNormalizationPlan)); } @@ -116,7 +124,7 @@ public class TestSimpleRegionNormalizer { setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue(plan instanceof EmptyNormalizationPlan); } @@ -147,7 +155,7 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri5.getRegionName(), 16); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue(plan instanceof MergeNormalizationPlan); assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion()); @@ -186,7 +194,7 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri6.getRegionName(), 2700); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue(plan instanceof MergeNormalizationPlan); assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion()); @@ -220,7 +228,7 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri5.getRegionName(), 5); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue(plan instanceof EmptyNormalizationPlan); } @@ -248,7 +256,7 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri4.getRegionName(), 30); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = normalizer.computePlanForTable(testTable, bothTypes); assertTrue(plan instanceof SplitNormalizationPlan); assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java index 4fe42ed..ce33b05 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.TableNamespaceManager; -import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.normalizer.NormalizationPlan.PlanType; import org.apache.hadoop.hbase.namespace.TestNamespaceAuditor; import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.regionserver.HRegion;