diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 8e51f25..798cdcc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1367,8 +1367,12 @@ public class HMaster extends HRegionServer implements MasterServices, Server { + " table or doesn't have auto normalization turned on"); continue; } - NormalizationPlan plan = this.normalizer.computePlanForTable(table); - plan.execute(clusterConnection.getAdmin()); + List plans = this.normalizer.computePlanForTable(table); + if (plans != null) { + for (NormalizationPlan plan : plans) { + plan.execute(clusterConnection.getAdmin()); + } + } } } // If Region did not generate any plans, it means the cluster is already balanced. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java index d60474d..5db6328 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java @@ -18,6 +18,8 @@ */ package org.apache.hadoop.hbase.master.normalizer; +import java.util.List; + import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; @@ -47,9 +49,9 @@ public interface RegionNormalizer { /** * Computes next optimal normalization plan. * @param table table to normalize - * @return Next (perhaps most urgent) normalization action to perform + * @return normalization actions to perform. Null if no action to take */ - NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException; + List computePlanForTable(TableName table) throws HBaseIOException; /** * Notification for the case where plan couldn't be executed due to constraint violation, such as diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java index 25118c7..c6baa2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerChore.java @@ -38,7 +38,7 @@ public class RegionNormalizerChore extends ScheduledChore { public RegionNormalizerChore(HMaster master) { super(master.getServerName() + "-RegionNormalizerChore", master, - master.getConfiguration().getInt("hbase.normalizer.period", 1800000)); + master.getConfiguration().getInt("hbase.normalizer.period", 300000)); this.master = master; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index fe10bd1..b2873bc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -71,22 +71,6 @@ public class SimpleRegionNormalizer implements RegionNormalizer { this.masterServices = masterServices; } - /* - * This comparator compares the region size. - * The second element in the triple is region size while the 3rd element - * is the index of the region in the underlying List - */ - private Comparator> regionSizeComparator = - new Comparator>() { - @Override - public int compare(Triple pair, - Triple pair2) { - long sz = pair.getSecond(); - long sz2 = pair2.getSecond(); - return (sz < sz2) ? -1 : ((sz == sz2) ? 0 : 1); - } - }; - @Override public void planSkipped(HRegionInfo hri, PlanType type) { skippedCount[type.ordinal()]++; @@ -97,6 +81,17 @@ public class SimpleRegionNormalizer implements RegionNormalizer { return skippedCount[type.ordinal()]; } + // Comparator that gives higher priority to region Split plan + private Comparator planComparator = + new Comparator() { + @Override + public int compare(NormalizationPlan plan, NormalizationPlan plan2) { + if (plan instanceof SplitNormalizationPlan) return -1; + if (plan2 instanceof SplitNormalizationPlan) return 1; + return 0; + } + }; + /** * Computes next most "urgent" normalization action on the table. * Action may be either a split, or a merge, or no action. @@ -105,12 +100,12 @@ public class SimpleRegionNormalizer implements RegionNormalizer { * @return normalization plan to execute */ @Override - public NormalizationPlan computePlanForTable(TableName table) throws HBaseIOException { + public List computePlanForTable(TableName table) throws HBaseIOException { if (table == null || table.isSystemTable()) { LOG.debug("Normalization of system table " + table + " isn't allowed"); - return EmptyNormalizationPlan.getInstance(); + return null; } - + List plans = new ArrayList(); List tableRegions = masterServices.getAssignmentManager().getRegionStates(). getRegionsOfTable(table); @@ -119,7 +114,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer { int nrRegions = tableRegions == null ? 0 : tableRegions.size(); LOG.debug("Table " + table + " has " + nrRegions + " regions, required min number" + " of regions for normalizer to run is " + MIN_REGION_COUNT + ", not running normalizer"); - return EmptyNormalizationPlan.getInstance(); + return null; } LOG.debug("Computing normalization plan for table: " + table + @@ -127,55 +122,48 @@ public class SimpleRegionNormalizer implements RegionNormalizer { long totalSizeMb = 0; - ArrayList> regionsWithSize = - new ArrayList>(tableRegions.size()); for (int i = 0; i < tableRegions.size(); i++) { HRegionInfo hri = tableRegions.get(i); long regionSize = getRegionSize(hri); - regionsWithSize.add(new Triple(hri, regionSize, i)); totalSizeMb += regionSize; } - Collections.sort(regionsWithSize, regionSizeComparator); - - Triple largestRegion = regionsWithSize.get(tableRegions.size()-1); - double avgRegionSize = totalSizeMb / (double) tableRegions.size(); LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb); LOG.debug("Table " + table + ", average region size: " + avgRegionSize); - // now; if the largest region is >2 times large than average, we split it, split - // is more high priority normalization action than merge. - if (largestRegion.getSecond() > 2 * avgRegionSize) { - LOG.debug("Table " + table + ", largest region " - + largestRegion.getFirst().getRegionNameAsString() + " has size " - + largestRegion.getSecond() + ", more than 2 times than avg size, splitting"); - return new SplitNormalizationPlan(largestRegion.getFirst(), null); - } int candidateIdx = 0; - // look for two successive entries whose indices are adjacent - while (candidateIdx < tableRegions.size()-1) { - if (Math.abs(regionsWithSize.get(candidateIdx).getThird() - - regionsWithSize.get(candidateIdx + 1).getThird()) == 1) { - break; + while (candidateIdx < tableRegions.size()) { + HRegionInfo hri = tableRegions.get(candidateIdx); + long regionSize = getRegionSize(hri); + // if the region is > 2 times larger than average, we split it, split + // is more high priority normalization action than merge. + if (regionSize > 2 * avgRegionSize) { + LOG.debug("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size " + + regionSize + ", more than twice avg size, splitting"); + plans.add(new SplitNormalizationPlan(hri, null)); + } else { + if (candidateIdx == tableRegions.size()-1) { + break; + } + HRegionInfo hri2 = tableRegions.get(candidateIdx+1); + long regionSize2 = getRegionSize(hri2); + if (regionSize + regionSize2 < avgRegionSize) { + LOG.debug("Table " + table + ", small region size: " + regionSize + + " plus its neighbor size: " + regionSize2 + + ", less than the avg size " + avgRegionSize + ", merging them"); + plans.add(new MergeNormalizationPlan(hri, hri2)); + candidateIdx++; + } } candidateIdx++; } - if (candidateIdx == tableRegions.size()-1) { - LOG.debug("No neighboring regions found for table: " + table); - return EmptyNormalizationPlan.getInstance(); - } - Triple candidateRegion = regionsWithSize.get(candidateIdx); - Triple candidateRegion2 = regionsWithSize.get(candidateIdx+1); - if (candidateRegion.getSecond() + candidateRegion2.getSecond() < avgRegionSize) { - LOG.debug("Table " + table + ", smallest region size: " + candidateRegion.getSecond() - + " and its smallest neighbor size: " + candidateRegion2.getSecond() - + ", less than the avg size, merging them"); - return new MergeNormalizationPlan(candidateRegion.getFirst(), - candidateRegion2.getFirst()); + if (plans.isEmpty()) { + LOG.debug("No normalization needed, regions look good for table: " + table); + return null; } - LOG.debug("No normalization needed, regions look good for table: " + table); - return EmptyNormalizationPlan.getInstance(); + Collections.sort(plans, planComparator); + return plans; } private long getRegionSize(HRegionInfo hri) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index 970af43..079f38a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -69,8 +69,8 @@ public class TestSimpleRegionNormalizer { Map regionSizes = new HashMap<>(); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); - assertTrue(plan instanceof EmptyNormalizationPlan); + List plans = normalizer.computePlanForTable(testTable); + assertTrue(plans == null); } @Test @@ -88,8 +88,8 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri2.getRegionName(), 15); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); - assertTrue((plan instanceof EmptyNormalizationPlan)); + List plans = normalizer.computePlanForTable(testTable); + assertTrue(plans == null); } @Test @@ -116,8 +116,8 @@ public class TestSimpleRegionNormalizer { setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); - assertTrue(plan instanceof EmptyNormalizationPlan); + List plans = normalizer.computePlanForTable(testTable); + assertTrue(plans == null); } @Test @@ -147,7 +147,8 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri5.getRegionName(), 16); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + List plans = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = plans.get(0); assertTrue(plan instanceof MergeNormalizationPlan); assertEquals(hri2, ((MergeNormalizationPlan) plan).getFirstRegion()); @@ -186,7 +187,8 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri6.getRegionName(), 2700); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + List plans = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = plans.get(0); assertTrue(plan instanceof MergeNormalizationPlan); assertEquals(hri5, ((MergeNormalizationPlan) plan).getFirstRegion()); @@ -220,9 +222,9 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri5.getRegionName(), 5); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + List plans = normalizer.computePlanForTable(testTable); - assertTrue(plan instanceof EmptyNormalizationPlan); + assertTrue(plans == null); } @Test @@ -248,7 +250,8 @@ public class TestSimpleRegionNormalizer { regionSizes.put(hri4.getRegionName(), 30); setupMocksForNormalizer(regionSizes, hris); - NormalizationPlan plan = normalizer.computePlanForTable(testTable); + List plans = normalizer.computePlanForTable(testTable); + NormalizationPlan plan = plans.get(0); assertTrue(plan instanceof SplitNormalizationPlan); assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java index 4fe42ed..4bcccc6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizerOnCluster.java @@ -155,11 +155,19 @@ public class TestSimpleRegionNormalizerOnCluster { } while (skippedSplitcnt == 0L); assert(skippedSplitcnt > 0); } else { - while (MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME) < 6) { - LOG.info("Waiting for normalization split to complete"); - Thread.sleep(100); + while (true) { + List regions = TEST_UTIL.getHBaseCluster().getRegions(TABLENAME); + int cnt = 0; + for (HRegion region : regions) { + String regionName = region.getRegionInfo().getRegionNameAsString(); + if (regionName.startsWith("testRegionNormalizationSplitOnCluster,zzzzz")) { + cnt++; + } + } + if (cnt >= 2) { + break; + } } - assertEquals(6, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLENAME)); } admin.disableTable(TABLENAME);