Index: src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (revision 1000714) +++ src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (working copy) @@ -106,21 +106,21 @@ // add a region server - total of 2 LOG.info("Started=" + cluster.startRegionServer().getRegionServer().getServerName()); - cluster.getMaster().balance(); + cluster.getMaster().balancer(); assertRegionsAreBalanced(); // add a region server - total of 3 LOG.debug("Adding 3rd region server."); LOG.info("Started=" + cluster.startRegionServer().getRegionServer().getServerName()); - cluster.getMaster().balance(); + cluster.getMaster().balancer(); assertRegionsAreBalanced(); // kill a region server - total of 2 LOG.debug("Killing the 3rd region server."); LOG.info("Stopped=" + cluster.stopRegionServer(2, false)); cluster.waitOnRegionServer(2); - cluster.getMaster().balance(); + cluster.getMaster().balancer(); assertRegionsAreBalanced(); // start two more region servers - total of 4 @@ -130,14 +130,14 @@ LOG.debug("Adding 4th region server"); LOG.info("Started=" + cluster.startRegionServer().getRegionServer().getServerName()); - cluster.getMaster().balance(); + cluster.getMaster().balancer(); assertRegionsAreBalanced(); for (int i = 0; i < 6; i++){ LOG.debug("Adding " + (i + 5) + "th region server"); cluster.startRegionServer(); } - cluster.getMaster().balance(); + cluster.getMaster().balancer(); assertRegionsAreBalanced(); } Index: src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java (revision 1000714) +++ src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java (working copy) @@ -60,7 +60,7 @@ LOG.info("before"); if (UTIL.ensureSomeRegionServersAvailable(2)) { // Distribute regions - UTIL.getMiniHBaseCluster().getMaster().balance(); + UTIL.getMiniHBaseCluster().getMaster().balancer(); } LOG.info("before done"); } Index: src/main/java/org/apache/hadoop/hbase/master/ServerManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (revision 1000715) +++ src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (working copy) @@ -84,6 +84,7 @@ private final Server master; private final MasterServices services; + private final boolean freshClusterStartup; private final ServerMonitor serverMonitorThread; @@ -119,10 +120,14 @@ * Constructor. * @param master * @param services + * @param freshClusterStartup True if we are original master on a fresh + * cluster startup else if false, we are joining an already running cluster. */ - public ServerManager(final Server master, final MasterServices services) { + public ServerManager(final Server master, final MasterServices services, + final boolean freshClusterStartup) { this.master = master; this.services = services; + this.freshClusterStartup = freshClusterStartup; Configuration c = master.getConfiguration(); int monitorInterval = c.getInt("hbase.master.monitor.interval", 60 * 1000); this.metrics = new MasterMetrics(master.getServerName()); @@ -249,9 +254,29 @@ // If we don't know this server, tell it shutdown. HServerInfo storedInfo = this.onlineServers.get(info.getServerName()); if (storedInfo == null) { - LOG.warn("Received report from unknown server -- telling it " + - "to " + HMsg.Type.STOP_REGIONSERVER + ": " + info.getServerName()); - return HMsg.STOP_REGIONSERVER_ARRAY; + if (!this.freshClusterStartup) { + // If we are joining an existing cluster, then soon as we come up we'll + // be getting reports from already running regionservers. + LOG.info("Registering new server: " + info.getServerName()); + // recordNewServer is what happens at the end of reportServerStartup. + // The only thing we are skipping is passing back to the regionserver + // the HServerInfo to use. Here we presume a master has already done + // that so we'll press on with whatever it gave us for HSI. + recordNewServer(info); + // If msgs, put off their processing but this is not enough because + // its possible that the next time the server reports in, we'll still + // not be up and serving. For example, if a split, we'll need the + // regions and servers setup in the master before the below + // handleSplitReport will work. TODO: FIx!! + if (msgs.length > 0) throw new PleaseHoldException("FIX! Putting off " + + "message processing because not yet rwady but possible we won't be " + + "ready next on next report"); + } else { + LOG.warn("Received report from unknown server, a server calling " + + " regionServerReport w/o having first called regionServerStartup; " + + "telling it " + HMsg.Type.STOP_REGIONSERVER + ": " + info.getServerName()); + return HMsg.STOP_REGIONSERVER_ARRAY; + } } // Check startcodes Index: src/main/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 1000715) +++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -241,7 +241,7 @@ this.connection = HConnectionManager.getConnection(conf); this.executorService = new ExecutorService(getServerName()); - this.serverManager = new ServerManager(this, this); + this.serverManager = new ServerManager(this, this, this.freshClusterStartup); this.catalogTracker = new CatalogTracker(this.zooKeeper, this.connection, this, conf.getInt("hbase.master.catalog.timeout", Integer.MAX_VALUE)); @@ -304,15 +304,13 @@ this.serverManager.waitForRegionServers(); // Start assignment of user regions, startup or failure - if (!this.stopped) { - if (this.freshClusterStartup) { - clusterStarterInitializations(this.fileSystemManager, - this.serverManager, this.catalogTracker, this.assignmentManager); - } else { - // Process existing unassigned nodes in ZK, read all regions from META, - // rebuild in-memory state. - this.assignmentManager.processFailover(); - } + if (this.freshClusterStartup) { + clusterStarterInitializations(this.fileSystemManager, + this.serverManager, this.catalogTracker, this.assignmentManager); + } else { + // Process existing unassigned nodes in ZK, read all regions from META, + // rebuild in-memory state. + this.assignmentManager.processFailover(); } // Start balancer and meta catalog janitor after meta and regions have @@ -320,6 +318,7 @@ this.balancerChore = getAndStartBalancerChore(this); this.catalogJanitorChore = Threads.setDaemonThreadRunning(new CatalogJanitor(this, this)); + // Check if we should stop every second. Sleeper sleeper = new Sleeper(1000, this); while (!this.stopped) sleeper.sleep(); @@ -442,9 +441,9 @@ try { // Start the executor service pools this.executorService.startExecutorService(ExecutorType.MASTER_OPEN_REGION, - conf.getInt("hbase.master.executor.openregion.threads", 5)); + conf.getInt("hbase.master.executor.openregion.threads", 10)); this.executorService.startExecutorService(ExecutorType.MASTER_CLOSE_REGION, - conf.getInt("hbase.master.executor.closeregion.threads", 5)); + conf.getInt("hbase.master.executor.closeregion.threads", 10)); this.executorService.startExecutorService(ExecutorType.MASTER_SERVER_OPERATIONS, conf.getInt("hbase.master.executor.serverops.threads", 5)); this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, @@ -496,14 +495,13 @@ } private static Thread getAndStartBalancerChore(final HMaster master) { - String name = master.getServerName() + "-balancerChore"; - int period = master.getConfiguration(). - getInt("hbase.balancer.period", 3000000); + String name = master.getServerName() + "-BalancerChore"; + int period = master.getConfiguration().getInt("hbase.balancer.period", 300000); // Start up the load balancer chore Chore chore = new Chore(name, period, master) { @Override protected void chore() { - master.balance(); + master.balancer(); } }; return Threads.setDaemonThreadRunning(chore); @@ -566,11 +564,8 @@ return !isStopped(); } - /** - * Run the balancer. - * @return True if balancer ran, false otherwise. - */ - public boolean balance() { + @Override + public boolean balancer() { // If balance not true, don't run balancer. if (!this.balance) return false; synchronized (this.balancer) { Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1000714) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -177,13 +177,13 @@ // Check existing regions in transition List nodes = ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.assignmentZNode); - if(nodes.isEmpty()) { + if (nodes.isEmpty()) { LOG.info("No regions in transition in ZK to process on failover"); return; } LOG.info("Failed-over master needs to process " + nodes.size() + " regions in transition"); - for(String regionName : nodes) { + for (String regionName: nodes) { RegionTransitionData data = ZKAssign.getData(watcher, regionName); HRegionInfo regionInfo = MetaReader.getRegion(catalogTracker, data.getRegionName()).getFirst(); @@ -738,10 +738,10 @@ private void rebuildUserRegions() throws IOException { Map allRegions = MetaReader.fullScan(catalogTracker); - for(Map.Entry region : allRegions.entrySet()) { + for (Map.Entry region : allRegions.entrySet()) { HServerAddress regionLocation = region.getValue(); HRegionInfo regionInfo = region.getKey(); - if(regionLocation == null) { + if (regionLocation == null) { regions.put(regionInfo, null); continue; } @@ -1020,6 +1020,7 @@ synchronized (this.regions) { checkRegion(hsi, parent, true); checkRegion(hsi, a, false); + checkRegion(hsi, b, false); this.regions.put(a, hsi); this.regions.put(b, hsi); removeFromServers(hsi, parent, true); @@ -1031,10 +1032,10 @@ } /* - * Caller must hold locks on regions Map. + * Caller must hold locks on this.regions Map. * @param hsi * @param hri - * @param expected + * @param expected True if we expect hri to be in this.regions. */ private void checkRegion(final HServerInfo hsi, final HRegionInfo hri, final boolean expected) { Index: src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (revision 1000714) +++ src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (working copy) @@ -77,7 +77,8 @@ *
  • Version 23: HBASE-2066, multi-put.
  • *
  • Version 24: HBASE-2473, create table with regions.
  • *
  • Version 25: Added openRegion and Stoppable/Abortable to API.
  • + *
  • Version 26: New master.
  • * */ - public static final long versionID = 25L; + public static final long versionID = 26L; } Index: src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (revision 1000714) +++ src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (working copy) @@ -144,6 +144,15 @@ throws UnknownRegionException; /** + * Invoke the balancer. Will run the balancer and if regions to move, it will + * go ahead and do the reassignments. Can NOT run for various reasons. Check + * logs. + * @return True if balancer ran, false otherwise. + */ + public boolean balancer(); + + /** + * Turn the load balancer on or off. * @param b If true, enable balancer. If false, disable balancer. * @return Previous balancer value */ Index: src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (revision 1000714) +++ src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (working copy) @@ -879,10 +879,9 @@ } /** + * Turn the load balancer on or off. * @param b If true, enable balancer. If false, disable balancer. * @return Previous balancer value - * @throws ZooKeeperConnectionException - * @throws MasterNotRunningException */ public boolean balance(final boolean b) throws MasterNotRunningException, ZooKeeperConnectionException { @@ -890,6 +889,17 @@ } /** + * Invoke the balancer. Will run the balancer and if regions to move, it will + * go ahead and do the reassignments. Can NOT run for various reasons. Check + * logs. + * @return True if balancer ran, false otherwise. + */ + public boolean balancer() + throws MasterNotRunningException, ZooKeeperConnectionException { + return getMaster().balancer(); + } + + /** * Split a table or an individual region. * Asynchronous operation. * Index: src/main/resources/hbase-default.xml =================================================================== --- src/main/resources/hbase-default.xml (revision 1000714) +++ src/main/resources/hbase-default.xml (working copy) @@ -157,10 +157,9 @@ hbase.regionserver.msginterval - 3000 + 5000 Interval between messages from the RegionServer to HMaster - in milliseconds. Use a high value for clusters with more than 100 - nodes. Default is 3 seconds. + in milliseconds.