Index: src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (revision 1202634) +++ src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java (working copy) @@ -132,6 +132,7 @@ // Updates from master to ZK. This is done by the master and there is // nothing to process by either Master or RS M_ZK_REGION_OFFLINE (50), // Master adds this region as offline in ZK + M_ZK_REGION_PENDING_CLOSE (51), // Master adds this region as pending close in ZK // Master controlled events to be executed on the master M_SERVER_SHUTDOWN (70), // Master is processing shutdown of a RS Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1202965) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -455,6 +455,15 @@ " in state " + data.getEventType()); synchronized (regionsInTransition) { switch (data.getEventType()) { + case M_ZK_REGION_PENDING_CLOSE: + + // insert region into RIT and trigger new unassign + regionsInTransition.put(encodedRegionName, new RegionState( + regionInfo, RegionState.State.PENDING_CLOSE, + data.getStamp(), data.getOrigin())); + invokeUnassign(regionInfo); + break; + case RS_ZK_REGION_CLOSING: // If zk node of the region was updated by a live server skip this // region and just add it into RIT. @@ -637,6 +646,9 @@ case M_ZK_REGION_OFFLINE: // Nothing to do. break; + case M_ZK_REGION_PENDING_CLOSE: + // Nothing to do. + break; case RS_ZK_REGION_SPLITTING: if (!isInStateForSplitting(regionState)) break; @@ -1743,7 +1755,7 @@ // Create the znode in CLOSING state try { - ZKAssign.createNodeClosing( + ZKAssign.createNodePendingClose( master.getZooKeeper(), region, master.getServerName()); } catch (KeeperException e) { master.abort("Unexpected ZK exception creating node CLOSING", e); @@ -1752,8 +1764,6 @@ state = new RegionState(region, RegionState.State.PENDING_CLOSE); regionsInTransition.put(encodedName, state); } else if (force && state.isPendingClose()) { - // JD 05/25/11 - // in my experience this is useless, when this happens it just spins debugLog(region, "Attempting to unassign region " + region.getRegionNameAsString() + " which is already pending close " + "but forcing an additional close"); @@ -2527,21 +2537,7 @@ LOG.info("Region has been PENDING_CLOSE for too " + "long, running forced unassign again on region=" + regionInfo.getRegionNameAsString()); - try { - // If the server got the RPC, it will transition the node - // to CLOSING, so only do something here if no node exists - if (!ZKUtil.watchAndCheckExists(watcher, - ZKAssign.getNodeName(watcher, regionInfo.getEncodedName()))) { - // Queue running of an unassign -- do actual unassign - // outside of the regionsInTransition lock. - invokeUnassign(regionInfo); - } - } catch (NoNodeException e) { - LOG.debug("Node no longer existed so not forcing another " - + "unassignment"); - } catch (KeeperException e) { - LOG.warn("Unexpected ZK exception timing out a region close", e); - } + invokeUnassign(regionInfo); break; case CLOSING: LOG.info("Region has been CLOSING for too " + Index: src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java (revision 1202965) +++ src/main/java/org/apache/hadoop/hbase/master/UnAssignCallable.java (working copy) @@ -40,7 +40,7 @@ @Override public Object call() throws Exception { - assignmentManager.unassign(hri); + assignmentManager.unassign(hri, true); return null; } } Index: src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java (revision 1202634) +++ src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java (working copy) @@ -25,7 +25,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.executor.EventHandler; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.zookeeper.ZKAssign; @@ -111,6 +113,16 @@ if (this.zk) { expectedVersion = getCurrentVersion(); if (expectedVersion == FAILED) return; + try { + expectedVersion = ZKAssign.transitionNodeClosing( + server.getZooKeeper(), regionInfo, this.server.getServerName(), + expectedVersion); + } catch (KeeperException e) { + LOG.error("Error transition from pendingclose to closing for region=" + + name, e); + return; + } + if (expectedVersion == FAILED) return; } // Close the region Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java (revision 1202634) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java (working copy) @@ -531,6 +531,41 @@ ZKUtil.deleteChildrenRecursively(zkw, zkw.assignmentZNode); } + /** + * Creates a new unassigned node in the PENDING_CLOSE state for the specified + * region. + * + *
Does not transition nodes from any states. If a node already exists + * for this region, a {@link NodeExistsException} will be thrown. + * + *
If creation is successful, returns the version number of the CLOSING + * node created. + * + *
Does not set any watches. + * + *
This method should only be used by a RegionServer when initiating a + * close of a region after receiving a CLOSE RPC from the Master. + * + * @param zkw zk reference + * @param region region to be created as pending close + * @param serverName server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException if unexpected zookeeper exception + * @throws KeeperException.NodeExistsException if node already exists + */ + public static int createNodePendingClose(ZooKeeperWatcher zkw, HRegionInfo region, + ServerName serverName) + throws KeeperException, KeeperException.NodeExistsException { + LOG.debug(zkw.prefix("Creating unassigned node for " + + region.getEncodedName() + " in a PENDING_CLOSE state")); + + RegionTransitionData data = new RegionTransitionData( + EventType.M_ZK_REGION_PENDING_CLOSE, region.getRegionName(), serverName); + + String node = getNodeName(zkw, region.getEncodedName()); + return ZKUtil.createAndWatch(zkw, node, data.getBytes()); + } + // RegionServer methods /** @@ -570,6 +605,49 @@ /** * Transitions an existing unassigned node for the specified region which is + * currently in the PENDING_CLOSE state to be in the CLOSING state. + * + *
+ * Does not transition nodes from other states. If for some reason the node + * could not be transitioned, the method returns -1. If the transition is + * successful, the version of the node after transition is returned. + * + *
+ * This method can fail and return false for three different reasons: + *
+ * Does not set any watches. + * + *
+ * This method should only be used by a RegionServer when initiating a close + * of a region after receiving a CLOSE RPC from the Master. + * + * @param zkw + * zk reference + * @param region + * region to be transitioned to closed + * @param serverName + * server event originates from + * @return version of node after transition, -1 if unsuccessful transition + * @throws KeeperException + * if unexpected zookeeper exception + */ + public static int transitionNodeClosing(ZooKeeperWatcher zkw, + HRegionInfo region, ServerName serverName, int expectedVersion) + throws KeeperException { + return transitionNode(zkw, region, serverName, + EventType.M_ZK_REGION_PENDING_CLOSE, EventType.RS_ZK_REGION_CLOSING, + expectedVersion); + } + + /** + * Transitions an existing unassigned node for the specified region which is * currently in the CLOSING state to be in the CLOSED state. * *
Does not transition nodes from other states. If for some reason the
Index: src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
===================================================================
--- src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java (revision 1202634)
+++ src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java (working copy)
@@ -378,11 +378,13 @@
// Let's just assign everything to first RS
HRegionServer hrs = cluster.getRegionServer(0);
ServerName serverName = hrs.getServerName();
-
+ HRegionInfo closingRegion = enabledRegions.remove(0);
// we'll need some regions to already be assigned out properly on live RS
List