diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index cfbeafc..a1cb28f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableNotFoundException; @@ -65,7 +66,6 @@ import org.apache.hadoop.hbase.executor.EventHandler; import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -import org.apache.hadoop.hbase.master.AssignmentManager.RegionState.State; import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; @@ -77,7 +77,6 @@ import org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; -import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.KeyLocker; import org.apache.hadoop.hbase.util.Pair; @@ -888,7 +887,10 @@ public class AssignmentManager extends ZooKeeperListener { regionState.update(RegionState.State.CLOSED, createTime, sn); // When there are more than one region server a new RS is selected as the // destination and the same is updated in the regionplan. (HBASE-5546) - getRegionPlan(regionState, sn, true); + RegionPlan regionPlan = getRegionPlan(regionState, sn, true); + if (isNullPlan(regionPlan, regionState, sn)) { + break; + } this.executorService.submit(new ClosedRegionHandler(master, this, regionState.getRegion())); break; @@ -1707,9 +1709,7 @@ public class AssignmentManager extends ZooKeeperListener { return; } RegionPlan plan = getRegionPlan(state, forceNewPlan); - if (plan == null) { - LOG.debug("Unable to determine a plan to assign " + state); - this.timeoutMonitor.setAllRegionServersOffline(true); + if (isNullPlan(plan, state, null)) { return; // Should get reassigned later when RIT times out. } try { @@ -1769,16 +1769,37 @@ public class AssignmentManager extends ZooKeeperListener { // Transition back to OFFLINE state.update(RegionState.State.OFFLINE); // Force a new plan and reassign. Will return null if no servers. - if (getRegionPlan(state, plan.getDestination(), true) == null) { - this.timeoutMonitor.setAllRegionServersOffline(true); - LOG.warn("Unable to find a viable location to assign region " + - state.getRegion().getRegionNameAsString()); - return; + RegionPlan newPlan = getRegionPlan(state, plan.getDestination(), true); + if (isNullPlan(newPlan, state, plan.getDestination())) { + // Whether no servers to assign too or region assignment is being handled elsewhere, + // skip out of this retry loop. + break; } } } } + /** + * Test if null plan. If it is, log appropriately. + * @param p + * @param state + * @param sn + * @return + */ + private boolean isNullPlan(final RegionPlan p, final RegionState state, final ServerName sn) { + if (p != null) return false; + String name = state.getRegion().getRegionNameAsString(); + if (getAssignableServers(sn) == null) { + // No plan because no servers to assign too + this.timeoutMonitor.setAllRegionServersOffline(true); + LOG.warn("Unable to find an assignable server for region " + name); + } else { + LOG.warn("No assignment plan for " + name + ", skipping; presume region will be " + + "assigned elsewhere (servershutdown handler or timeout monitor?)"); + } + return true; + } + private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) { String tableName = region.getTableNameAsString(); boolean disabled = this.zkTable.isDisabledTable(tableName); @@ -1873,73 +1894,76 @@ public class AssignmentManager extends ZooKeeperListener { } /** + * Returns list of servers we can assign too. It is made of the current list of servers + * minus the passed serversToExclude and any dead servers. + * @param serverToExclude + * @return Null if no servers to assign too or list of assignable servers. + */ + List getAssignableServers(final ServerName serverToExclude) { + // This destServers should not include servers that are currently being processed by + // server shutdown handler, etc. Dead servers are excluded from the list returned. + final List destServers = + this.serverManager.createDestinationServersList(serverToExclude); + return destServers.isEmpty()? null: destServers; + } + + /** * @param state * @return Plan for passed state (If none currently, it creates one or * if no servers to assign, it returns null). */ - RegionPlan getRegionPlan(final RegionState state, - final boolean forceNewPlan) { + RegionPlan getRegionPlan(final RegionState state, final boolean forceNewPlan) { return getRegionPlan(state, null, forceNewPlan); } /** + * Get an assignment plan for passed in region. * @param state * @param serverToExclude Server to exclude (we know its bad). Pass null if * all servers are thought to be assignable. * @param forceNewPlan If true, then if an existing plan exists, a new plan * will be generated. - * @return Plan for passed state (If none currently, it creates one or - * if no servers to assign, it returns null). + * @return Plan for passed state (If none currently, it creates one) or null if no + * servers to assign too or if current plan is for a server that is dead (if for a server that is + * dead, the region will be handled over in the server shutdown handler code so don't assign in + * here). */ - RegionPlan getRegionPlan(final RegionState state, - final ServerName serverToExclude, final boolean forceNewPlan) { - // Pickup existing plan or make a new one + RegionPlan getRegionPlan(final RegionState state, final ServerName serverToExclude, + final boolean forceNewPlan) { final String encodedName = state.getRegion().getEncodedName(); - final List destServers = - serverManager.createDestinationServersList(serverToExclude); - - if (destServers.isEmpty()){ - LOG.warn("Can't move the region " + encodedName + - ", there is no destination server available."); + final List destServers = getAssignableServers(serverToExclude); + if (destServers == null) { + LOG.warn("Can't move region=" + encodedName + ", there is no destination server available."); return null; } - RegionPlan randomPlan = null; - boolean newPlan = false; RegionPlan existingPlan = null; - synchronized (this.regionPlans) { existingPlan = this.regionPlans.get(encodedName); - if (existingPlan != null && existingPlan.getDestination() != null) { - LOG.debug("Found an existing plan for " + - state.getRegion().getRegionNameAsString() + - " destination server is " + existingPlan.getDestination().toString()); + LOG.debug("Found an existing plan for " + state.getRegion().getRegionNameAsString() + + " destination server is " + existingPlan.getDestination().toString()); } - if (forceNewPlan || existingPlan == null || existingPlan.getDestination() == null || !destServers.contains(existingPlan.getDestination())) { - newPlan = true; randomPlan = new RegionPlan(state.getRegion(), null, - balancer.randomAssignment(state.getRegion(), destServers)); + this.balancer.randomAssignment(state.getRegion(), destServers)); this.regionPlans.put(encodedName, randomPlan); } } - - if (newPlan) { - LOG.debug("No previous transition plan was found (or we are ignoring " + - "an existing plan) for " + state.getRegion().getRegionNameAsString() + - " so generated a random one; " + randomPlan + "; " + - serverManager.countOfRegionServers() + - " (online=" + serverManager.getOnlineServers().size() + - ", available=" + destServers.size() + ") available servers"); - return randomPlan; - } - LOG.debug("Using pre-existing plan for region " + - state.getRegion().getRegionNameAsString() + "; plan=" + existingPlan); - return existingPlan; + if (randomPlan != null) { + LOG.debug("No previous transition plan was found (or we are ignoring an existing plan) for " + + state.getRegion().getRegionNameAsString() + " so generated a random one; " + randomPlan + + "; " + this.serverManager.countOfRegionServers() + " (online=" + + this.serverManager.getOnlineServers().size() + ", available=" + destServers.size() + + ") available servers"); + return randomPlan; + } + LOG.debug("Using pre-existing plan for region " + state.getRegion().getRegionNameAsString() + + "; plan=" + existingPlan); + return existingPlan; } /** @@ -3197,11 +3221,15 @@ public class AssignmentManager extends ZooKeeperListener { } /** - * Process shutdown server removing any assignments. + * Start processing of shutdown server. + * When server shutdown handling is done, call {@link #processServerShutdownFinish(ServerName)} * @param sn Server that went down. - * @return list of regions in transition on this server + * @return Pair that has all regionplans that pertain to this dead server and a list that has + * the overlap of regions in transition and regions that were on the server. + * @see #processServerShutdownFinish(ServerName) */ - public List processServerShutdown(final ServerName sn) { + public Pair, List> processServerShutdown(final ServerName sn) { + Set regionPlans = new ConcurrentSkipListSet(); // Clean out any existing assignment plans for this server synchronized (this.regionPlans) { for (Iterator > i = @@ -3211,6 +3239,7 @@ public class AssignmentManager extends ZooKeeperListener { // The name will be null if the region is planned for a random assign. if (otherSn != null && otherSn.equals(sn)) { // Use iterator's remove else we'll get CME + regionPlans.add(e.getValue().getRegionInfo()); i.remove(); } } @@ -3224,7 +3253,7 @@ public class AssignmentManager extends ZooKeeperListener { Set assignedRegions = this.servers.remove(sn); if (assignedRegions == null || assignedRegions.isEmpty()) { // No regions on this server, we are done, return empty list of RITs - return rits; + return new Pair, List>(regionPlans, rits); } deadRegions = new TreeSet(assignedRegions); for (HRegionInfo region : deadRegions) { @@ -3241,7 +3270,7 @@ public class AssignmentManager extends ZooKeeperListener { rits.add(region); } } - return rits; + return new Pair, List>(regionPlans, rits); } /** @@ -3656,4 +3685,29 @@ public class AssignmentManager extends ZooKeeperListener { this.master.abort(errorMsg, e); } } -} + + /** + * + * @param hri + * @return whether region is online or not. + */ + public boolean isRegionOnline(HRegionInfo hri) { + ServerName sn = null; + synchronized (this.regions) { + sn = this.regions.get(hri); + if (sn == null) { + return false; + } + if (this.isServerOnline(sn)) { + return true; + } + // Remove the assignment mapping for sn. + Set hriSet = this.servers.get(sn); + if (hriSet != null) { + hriSet.remove(hri); + } + this.regions.remove(hri); + return false; + } + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index ba1348e..69d90c5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -742,7 +742,7 @@ public class ServerManager { } } } - + /** * To clear any dead server with same host name and port of any online server */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java index 00f0407..8f189ed 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.NavigableMap; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,6 +44,7 @@ import org.apache.hadoop.hbase.master.DeadServer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.zookeeper.KeeperException; /** @@ -228,13 +230,12 @@ public class ServerShutdownHandler extends EventHandler { return; } - // Clean out anything in regions in transition. Being conservative and - // doing after log splitting. Could do some states before -- OPENING? - // OFFLINE? -- and then others after like CLOSING that depend on log - // splitting. - List regionsInTransition = - this.services.getAssignmentManager(). - processServerShutdown(this.serverName); + // Returns set of regions that had regionplans against the downed server and a list of + // the intersection of regions-in-transition and regions that were on the server that died. + Pair, List> p = + this.services.getAssignmentManager().processServerShutdown(this.serverName); + Set outstandingRegionPlans = p.getFirst(); + List ritsIntersection = p.getSecond(); // Wait on meta to come online; we need it to progress. // TODO: Best way to hold strictly here? We should build this retry logic @@ -268,7 +269,7 @@ public class ServerShutdownHandler extends EventHandler { } // Skip regions that were in transition unless CLOSING or PENDING_CLOSE - for (RegionState rit : regionsInTransition) { + for (RegionState rit : ritsIntersection) { if (!rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()) { LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() + " from list of regions to assign because in RIT; region state: " + @@ -277,16 +278,14 @@ public class ServerShutdownHandler extends EventHandler { } } - assert regionsInTransition != null; - LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) + - " region(s) that " + (serverName == null? "null": serverName) + - " was carrying (skipping " + - regionsInTransition.size() + - " regions(s) that are already in transition)"); + assert ritsIntersection != null; + LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) + " region(s) that " + + (serverName == null? "null": serverName) + " was carrying (skipping " + + ritsIntersection.size() + " regions(s) that are already in transition)"); // Iterate regions that were on this server and assign them + List toAssignRegions = new ArrayList(); if (hris != null) { - List toAssignRegions = new ArrayList(); for (Map.Entry e: hris.entrySet()) { RegionState rit = this.services.getAssignmentManager().isRegionInTransition(e.getKey()); if (processDeadRegion(e.getKey(), e.getValue(), @@ -294,19 +293,17 @@ public class ServerShutdownHandler extends EventHandler { this.server.getCatalogTracker())) { ServerName addressFromAM = this.services.getAssignmentManager() .getRegionServerOfRegion(e.getKey()); - if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()) { + if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting() + && !outstandingRegionPlans.contains(rit.getRegion())) { // Skip regions that were in transition unless CLOSING or - // PENDING_CLOSE + // PENDING_CLOSE or splitting LOG.info("Skip assigning region " + rit.toString()); - } else if (addressFromAM != null - && !addressFromAM.equals(this.serverName)) { - LOG.debug("Skip assigning region " - + e.getKey().getRegionNameAsString() - + " because it has been opened in " - + addressFromAM.getServerName()); - } else { - toAssignRegions.add(e.getKey()); - } + } else if (addressFromAM != null && !addressFromAM.equals(this.serverName)) { + LOG.debug("Skip assigning region " + e.getKey().getRegionNameAsString() + + " because it has been opened in " + addressFromAM.getServerName()); + } else { + toAssignRegions.add(e.getKey()); + } } else if (rit != null && (rit.isSplitting() || rit.isSplit())) { // This will happen when the RS went down and the call back for the SPLIITING or SPLIT // has not yet happened for node Deleted event. In that case if the region was actually @@ -335,12 +332,21 @@ public class ServerShutdownHandler extends EventHandler { toAssignRegions.remove(hri); } } - // Get all available servers - List availableServers = services.getServerManager() - .createDestinationServersList(); - this.services.getAssignmentManager().assign(toAssignRegions, - availableServers); } + int reassignedRegions = 0; + // Get all available servers + for (HRegionInfo hri : outstandingRegionPlans) { + if (!this.services.getAssignmentManager().isRegionOnline(hri)) { + if (!toAssignRegions.contains(hri)) { + toAssignRegions.add(hri); + } + } + } + List availableServers = services.getServerManager() + .createDestinationServersList(); + this.services.getAssignmentManager().assign(toAssignRegions, availableServers); + LOG.info(reassignedRegions + " regions which were planned to open on " + this.serverName + + " have been re-assigned."); } finally { this.deadServers.finish(serverName); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java index d8b8548..5966f27 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java @@ -25,8 +25,10 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.hbase.DeserializationException; @@ -34,6 +36,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.Server; @@ -398,7 +401,7 @@ public class TestAssignmentManager { AssignmentManager am = new AssignmentManager(this.server, this.serverManager, ct, balancer, executor, null); try { - processServerShutdownHandler(ct, am, false); + processServerShutdownHandler(ct, am, false, null); } finally { executor.shutdown(); am.shutdown(); @@ -464,7 +467,7 @@ public class TestAssignmentManager { ZKUtil.createAndWatch(this.watcher, node, data.toByteArray()); try { - processServerShutdownHandler(ct, am, regionSplitDone); + processServerShutdownHandler(ct, am, regionSplitDone, null); // check znode deleted or not. // In both cases the znode should be deleted. @@ -517,7 +520,7 @@ public class TestAssignmentManager { // create znode in M_ZK_REGION_CLOSING state. ZKUtil.createAndWatch(this.watcher, node, data.toByteArray()); try { - processServerShutdownHandler(ct, am, false); + processServerShutdownHandler(ct, am, false, null); // check znode deleted or not. // In both cases the znode should be deleted. assertTrue("The znode should be deleted.", ZKUtil.checkExists(this.watcher, node) == -1); @@ -536,8 +539,116 @@ public class TestAssignmentManager { ZKAssign.deleteAllNodes(this.watcher); } } + + /** + * + * Test verifies if region assignment through balancer is skipped when SSH processing a region. + * See HBASE-5816. + */ + @Test + public void testAssignThroughBalancerOrFailedOpenAndSSHInParallel() throws IOException, + KeeperException, ServiceException, InterruptedException { + try { + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + MockedLoadBalancer.class, LoadBalancer.class); + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, + this.serverManager); + // Boolean variable used for waiting until randomAssignment is called and + // new + // plan is generated. + AtomicBoolean gate = new AtomicBoolean(false); + if (balancer instanceof MockedLoadBalancer) { + ((MockedLoadBalancer) balancer).setGateVariable(gate); + } + ZKAssign.deleteAllNodes(this.watcher); + ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); + int v = ZKAssign.getVersion(this.watcher, REGIONINFO); + ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, + EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v); + String path = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName()); + RegionState state = new RegionState(REGIONINFO, State.OPENING, System.currentTimeMillis(), + SERVERNAME_B); + am.regionsInTransition.put(REGIONINFO.getEncodedName(), state); + // a dummy plan inserted into the regionPlans. This plan is cleared and + // new one is formed + am.regionPlans.put(REGIONINFO.getEncodedName(), + new RegionPlan(REGIONINFO, null, SERVERNAME_B)); + List serverList = new ArrayList(2); + serverList.add(SERVERNAME_B); + Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(serverList); + // In SSH we populate regions from region plans of dead server. See HBASE-5396 + addRegionToDeadServerRegions(am); + am.nodeDataChanged(path); + am.regionPlans.put(REGIONINFO.getEncodedName(), + new RegionPlan(REGIONINFO, null, SERVERNAME_B)); + processServerShutdownHandler(ct, am, false, SERVERNAME_B); + + // new region plan may take some time to get updated after random + // assignment is called and gate is set to true. + RegionPlan newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + while (newRegionPlan == null) { + Thread.sleep(10); + newRegionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); + } + // The new region plan should not be used because region assignment is in progress in SSH. + assertTrue("Assign should be invoked.", am.assignInvoked); + + } finally { + this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + DefaultLoadBalancer.class, LoadBalancer.class); + // Clean up all znodes + ZKAssign.deleteAllNodes(this.watcher); + + } + } + + private void addRegionToDeadServerRegions(AssignmentManagerWithExtrasForTesting am) { + Set regionsSet = new HashSet(1); + regionsSet.add(REGIONINFO); + RegionsOnDeadServer rds = new RegionsOnDeadServer(); + rds.setRegionsFromRegionPlansForServer(regionsSet); + am.getDeadServerRegionsFromRegionPlanForTest().put(SERVERNAME_A, rds); + } + + + /** + * When region in transition if region server opening the region gone down then region assignment + * taking long time(Waiting for timeout monitor to trigger assign). HBASE-5396(HBASE-6060) fixes this + * scenario. This test case verifies whether SSH calling assign for the region in transition or not. + * + * @throws KeeperException + * @throws IOException + * @throws ServiceException + */ + @Test + public void testSSHWhenSourceRSandDestRSInRegionPlanGoneDown() throws KeeperException, IOException, + ServiceException { + // We need a mocked catalog tracker. + CatalogTracker ct = Mockito.mock(CatalogTracker.class); + // Create an AM. + AssignmentManagerWithExtrasForTesting am = + setUpMockedAssignmentManager(this.server, this.serverManager); + // adding region in pending open. + am.regionsInTransition.put(REGIONINFO.getEncodedName(), new RegionState(REGIONINFO, + State.PENDING_OPEN)); + // adding region plan + am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B)); + am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString()); + + try { + processServerShutdownHandler(ct, am, false, SERVERNAME_A); + processServerShutdownHandler(ct, am, false, SERVERNAME_B); + assertTrue("Assign should be invoked.", am.assignInvoked); + } finally { + am.regionsInTransition.remove(REGIONINFO.getEncodedName()); + am.regionPlans.remove(REGIONINFO.getEncodedName()); + } + + } - private void processServerShutdownHandler(CatalogTracker ct, AssignmentManager am, boolean splitRegion) + private void processServerShutdownHandler(CatalogTracker ct, + AssignmentManager am, boolean splitRegion, ServerName sn) throws IOException, ServiceException { // Make sure our new AM gets callbacks; once registered, can't unregister. // Thats ok because we make a new zk watcher for each test. @@ -549,10 +660,19 @@ public class TestAssignmentManager { // Get a meta row result that has region up on SERVERNAME_A Result r = null; - if (splitRegion) { - r = Mocking.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A); + if (sn == null) { + if (splitRegion) { + r = Mocking + .getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A); + } else { + r = Mocking.getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + } } else { - r = Mocking.getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + if (sn.equals(SERVERNAME_A)) { + r = Mocking.getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + } else if (sn.equals(SERVERNAME_B)) { + r = new Result(new KeyValue[0]); + } } ScanResponse.Builder builder = ScanResponse.newBuilder(); @@ -580,8 +700,12 @@ public class TestAssignmentManager { MasterServices services = Mockito.mock(MasterServices.class); Mockito.when(services.getAssignmentManager()).thenReturn(am); Mockito.when(services.getServerManager()).thenReturn(this.serverManager); - ServerShutdownHandler handler = new ServerShutdownHandler(this.server, - services, deadServers, SERVERNAME_A, false); + ServerShutdownHandler handler = null; + if (sn != null) { + handler = new ServerShutdownHandler(this.server, services, deadServers, sn, false); + } else { + handler = new ServerShutdownHandler(this.server, services, deadServers, SERVERNAME_A, false); + } handler.process(); // The region in r will have been assigned. It'll be up in zk as unassigned. } @@ -943,8 +1067,13 @@ public class TestAssignmentManager { @Override public void assign(java.util.List regions, java.util.List servers) { - assignInvoked = true; + if (!regions.isEmpty()) { + assignInvoked = true; + } else { + assignInvoked = false; + } }; + /** reset the watcher */ void setWatcher(ZooKeeperWatcher watcher) { this.watcher = watcher;