diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index ba82aa3..9255a09 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -660,6 +660,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.markComplete("Initialization successful"); LOG.info("Master has completed initialization"); + configurationManager.registerObserver(this.balancer); initialized = true; // clear the dead servers with same host name and port of online server because we are not // removing dead server with same hostname and port of rs which is trying to check in before diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java index e24d745..f979403 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java @@ -22,7 +22,9 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HRegionInfo; @@ -46,7 +48,7 @@ import org.apache.hadoop.hbase.Stoppable; *

This classes produces plans for the {@link AssignmentManager} to execute. */ @InterfaceAudience.Private -public interface LoadBalancer extends Configurable, Stoppable { +public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObserver { /** * Set the current cluster status. This allows a LoadBalancer to map host name to a server @@ -130,4 +132,10 @@ public interface LoadBalancer extends Configurable, Stoppable { * @param regionInfo */ void regionOffline(HRegionInfo regionInfo); + + /* + * Notification that config has changed + * @param conf + */ + void onConfigurationChange(Configuration conf); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java index 1385444..d8dfbd0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.RegionLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.conf.ConfigurationObserver; import org.apache.hadoop.hbase.master.LoadBalancer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RackManager; @@ -1388,4 +1389,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer { return new HashMap>(); } } + + @Override + public void onConfigurationChange(Configuration conf) { + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 8187630..7c3e0e5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -125,7 +125,12 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { private RegionReplicaRackCostFunction regionReplicaRackCostFunction; @Override - public void setConf(Configuration conf) { + public void onConfigurationChange(Configuration conf) { + setConf(conf); + } + + @Override + public synchronized void setConf(Configuration conf) { super.setConf(conf); maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps); @@ -135,15 +140,19 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember); - localityCandidateGenerator = new LocalityBasedCandidateGenerator(services); + if (localityCandidateGenerator == null) { + localityCandidateGenerator = new LocalityBasedCandidateGenerator(services); + } localityCost = new LocalityCostFunction(conf, services); - candidateGenerators = new CandidateGenerator[] { - new RandomCandidateGenerator(), - new LoadCandidateGenerator(), - localityCandidateGenerator, - new RegionReplicaRackCandidateGenerator(), - }; + if (candidateGenerators == null) { + candidateGenerators = new CandidateGenerator[] { + new RandomCandidateGenerator(), + new LoadCandidateGenerator(), + localityCandidateGenerator, + new RegionReplicaRackCandidateGenerator(), + }; + } regionLoadFunctions = new CostFromRegionLoadFunction[] { new ReadRequestCostFunction(conf), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 7169867..41ea09d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -442,7 +442,7 @@ public class HRegionServer extends HasThread implements * Configuration manager is used to register/deregister and notify the configuration observers * when the regionserver is notified that there was a change in the on disk configs. */ - private final ConfigurationManager configurationManager; + protected final ConfigurationManager configurationManager; /** * Starts a HRegionServer at the default location. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java index 178bc53..73e493b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java @@ -18,10 +18,17 @@ */ package org.apache.hadoop.hbase.client; +import static org.junit.Assert.assertEquals; + import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -46,4 +53,25 @@ public class TestUpdateConfiguration { ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName(); admin.updateConfiguration(server); } + + @Test + public void testMasterOnlineConfigChange() throws IOException { + LOG.debug("Starting the test"); + Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml"); + Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml"); + Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml"); + // make a backup of hbase-site.xml + Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING); + // update hbase-site.xml by overwriting it + Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING); + + Admin admin = TEST_UTIL.getHBaseAdmin(); + ServerName server = TEST_UTIL.getHBaseCluster().getMaster().getServerName(); + admin.updateConfiguration(server); + Configuration conf = TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration(); + int custom = conf.getInt("hbase.custom.config", 0); + assertEquals(custom, 1000); + // restore hbase-site.xml + Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING); + } } diff --git a/hbase-server/src/test/resources/hbase-site2.xml b/hbase-server/src/test/resources/hbase-site2.xml new file mode 100644 index 0000000..87d340e --- /dev/null +++ b/hbase-server/src/test/resources/hbase-site2.xml @@ -0,0 +1,154 @@ + + + + + + hbase.custom.config + 1000 + + + hbase.regionserver.msginterval + 1000 + Interval between messages from the RegionServer to HMaster + in milliseconds. Default is 15. Set this value low if you want unit + tests to be responsive. + + + + hbase.defaults.for.version.skip + true + + + hbase.server.thread.wakefrequency + 1000 + Time to sleep in between searches for work (in milliseconds). + Used as sleep interval by service threads such as hbase:meta scanner and log roller. + + + + hbase.master.event.waiting.time + 50 + Time to sleep between checks to see if a table event took place. + + + + hbase.regionserver.handler.count + 5 + + + hbase.master.info.port + -1 + The port for the hbase master web UI + Set to -1 if you do not want the info server to run. + + + + hbase.master.port + 0 + Always have masters and regionservers come up on port '0' so we don't clash over + default ports. + + + + hbase.regionserver.port + 0 + Always have masters and regionservers come up on port '0' so we don't clash over + default ports. + + + + hbase.ipc.client.fallback-to-simple-auth-allowed + true + + + + hbase.regionserver.info.port + -1 + The port for the hbase regionserver web UI + Set to -1 if you do not want the info server to run. + + + + hbase.regionserver.info.port.auto + true + Info server auto port bind. Enables automatic port + search if hbase.regionserver.info.port is already in use. + Enabled for testing to run multiple tests on one machine. + + + + hbase.master.lease.thread.wakefrequency + 3000 + The interval between checks for expired region server leases. + This value has been reduced due to the other reduced values above so that + the master will notice a dead region server sooner. The default is 15 seconds. + + + + hbase.regionserver.safemode + false + + Turn on/off safe mode in region server. Always on for production, always off + for tests. + + + + hbase.hregion.max.filesize + 67108864 + + Maximum desired file size for an HRegion. If filesize exceeds + value + (value / 2), the HRegion is split in two. Default: 256M. + + Keep the maximum filesize small so we split more often in tests. + + + + hadoop.log.dir + ${user.dir}/../logs + + + hbase.zookeeper.property.clientPort + 21818 + Property from ZooKeeper's config zoo.cfg. + The port at which the clients will connect. + + + + hbase.defaults.for.version.skip + true + + Set to true to skip the 'hbase.defaults.for.version'. + Setting this to true can be useful in contexts other than + the other side of a maven generation; i.e. running in an + ide. You'll want to set this boolean to true to avoid + seeing the RuntimException complaint: "hbase-default.xml file + seems to be for and old version of HBase (@@@VERSION@@@), this + version is X.X.X-SNAPSHOT" + + + + hbase.table.sanity.checks + false + Skip sanity checks in tests + + +