diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 142502a..7942b28 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -583,6 +583,7 @@ public class HMaster extends HRegionServer implements MasterServices { this.balancer = LoadBalancerFactory.getLoadBalancer(conf); this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf); this.normalizer.setMasterServices(this); + this.normalizer.setMasterRpcServices((MasterRpcServices)rpcServices); this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this); this.loadBalancerTracker.start(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java index 0ee25f4..5b9f592 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizer.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; @@ -47,6 +48,13 @@ public interface RegionNormalizer { void setMasterServices(MasterServices masterServices); /** + * Set the master RPC service. Must be called before first call to + * {@link #computePlanForTable(TableName)}. + * @param masterRpcServices master RPC services to use + */ + void setMasterRpcServices(MasterRpcServices masterRpcServices); + + /** * Computes next optimal normalization plan. * @param table table to normalize * @return normalization actions to perform. Null if no action to take diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index a99f8dd..c321780 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -26,8 +26,13 @@ import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Admin.MasterSwitchType; +import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.normalizer.NormalizationPlan.PlanType; +import org.apache.hadoop.hbase.protobuf.RequestConverter; + +import com.google.protobuf.ServiceException; import java.util.ArrayList; import java.util.Collections; @@ -59,6 +64,7 @@ public class SimpleRegionNormalizer implements RegionNormalizer { private static final Log LOG = LogFactory.getLog(SimpleRegionNormalizer.class); private static final int MIN_REGION_COUNT = 3; private MasterServices masterServices; + private MasterRpcServices masterRpcServices; private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length]; /** @@ -71,6 +77,11 @@ public class SimpleRegionNormalizer implements RegionNormalizer { } @Override + public void setMasterRpcServices(MasterRpcServices masterRpcServices) { + this.masterRpcServices = masterRpcServices; + } + + @Override public void planSkipped(HRegionInfo hri, PlanType type) { skippedCount[type.ordinal()]++; } @@ -138,27 +149,44 @@ public class SimpleRegionNormalizer implements RegionNormalizer { LOG.debug("Table " + table + ", average region size: " + avgRegionSize); int candidateIdx = 0; + boolean splitEnabled = true, mergeEnabled = true; + try { + splitEnabled = masterRpcServices.isSplitOrMergeEnabled(null, + RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.SPLIT)).getEnabled(); + } catch (ServiceException se) { + LOG.debug("Unable to determine whether split is enabled", se); + } + try { + mergeEnabled = masterRpcServices.isSplitOrMergeEnabled(null, + RequestConverter.buildIsSplitOrMergeEnabledRequest(MasterSwitchType.MERGE)).getEnabled(); + } catch (ServiceException se) { + LOG.debug("Unable to determine whether split is enabled", se); + } while (candidateIdx < tableRegions.size()) { HRegionInfo hri = tableRegions.get(candidateIdx); long regionSize = getRegionSize(hri); // if the region is > 2 times larger than average, we split it, split // is more high priority normalization action than merge. if (regionSize > 2 * avgRegionSize) { - LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size " - + regionSize + ", more than twice avg size, splitting"); - plans.add(new SplitNormalizationPlan(hri, null)); + if (splitEnabled) { + LOG.info("Table " + table + ", large region " + hri.getRegionNameAsString() + " has size " + + regionSize + ", more than twice avg size, splitting"); + plans.add(new SplitNormalizationPlan(hri, null)); + } } else { if (candidateIdx == tableRegions.size()-1) { break; } - HRegionInfo hri2 = tableRegions.get(candidateIdx+1); - long regionSize2 = getRegionSize(hri2); - if (regionSize + regionSize2 < avgRegionSize) { - LOG.info("Table " + table + ", small region size: " + regionSize - + " plus its neighbor size: " + regionSize2 - + ", less than the avg size " + avgRegionSize + ", merging them"); - plans.add(new MergeNormalizationPlan(hri, hri2)); - candidateIdx++; + if (mergeEnabled) { + HRegionInfo hri2 = tableRegions.get(candidateIdx+1); + long regionSize2 = getRegionSize(hri2); + if (regionSize + regionSize2 < avgRegionSize) { + LOG.info("Table " + table + ", small region size: " + regionSize + + " plus its neighbor size: " + regionSize2 + + ", less than the avg size " + avgRegionSize + ", merging them"); + plans.add(new MergeNormalizationPlan(hri, hri2)); + candidateIdx++; + } } } candidateIdx++; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index babf22d..9d171f0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -25,7 +25,11 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.MasterRpcServices; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.protobuf.RequestConverter; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledResponse; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -34,6 +38,9 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -56,6 +63,7 @@ public class TestSimpleRegionNormalizer { // mocks private static MasterServices masterServices; + private static MasterRpcServices masterRpcServices; @BeforeClass public static void beforeAllTests() throws Exception { @@ -259,6 +267,7 @@ public class TestSimpleRegionNormalizer { protected void setupMocksForNormalizer(Map regionSizes, List hris) { masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS); + masterRpcServices = Mockito.mock(MasterRpcServices.class, RETURNS_DEEP_STUBS); // for simplicity all regions are assumed to be on one server; doesn't matter to us ServerName sn = ServerName.valueOf("localhost", 0, 1L); @@ -275,7 +284,15 @@ public class TestSimpleRegionNormalizer { when(masterServices.getServerManager().getLoad(sn). getRegionsLoad().get(region.getKey())).thenReturn(regionLoad); } + try { + when(masterRpcServices.isSplitOrMergeEnabled(any(RpcController.class), + any(IsSplitOrMergeEnabledRequest.class))).thenReturn( + IsSplitOrMergeEnabledResponse.newBuilder().setEnabled(true).build()); + } catch (ServiceException se) { + LOG.debug("error setting isSplitOrMergeEnabled switch", se); + } normalizer.setMasterServices(masterServices); + normalizer.setMasterRpcServices(masterRpcServices); } }