diff --git src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index c3420e5..b67e9ff 100644 --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -62,7 +62,6 @@ import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.RegionTransitionData; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; -import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; @@ -84,7 +83,7 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.ipc.RemoteException; import org.apache.zookeeper.AsyncCallback; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; /** @@ -246,6 +245,8 @@ public class AssignmentManager extends ZooKeeperListener { /** * Add a regionPlan for the specified region. + * @param encodedName + * @param plan */ public void addPlan(String encodedName, RegionPlan plan) { synchronized (regionPlans) { @@ -405,7 +406,7 @@ public class AssignmentManager extends ZooKeeperListener { boolean intransistion = processRegionInTransition(hri.getEncodedName(), hri, null); if (!intransistion) return intransistion; - debugLog(hri, "Waiting on " + HRegionInfo.prettyPrint(hri.getEncodedName())); + LOG.debug("Waiting on " + HRegionInfo.prettyPrint(hri.getEncodedName())); synchronized(this.regionsInTransition) { while (!this.master.isStopped() && this.regionsInTransition.containsKey(hri.getEncodedName())) { @@ -542,7 +543,7 @@ public class AssignmentManager extends ZooKeeperListener { throws KeeperException { // If was on dead server, its closed now. Force to OFFLINE and then // handle it like a close; this will get it reassigned if appropriate - debugLog(hri, "RIT " + hri.getEncodedName() + " in state=" + + LOG.debug("RIT " + hri.getEncodedName() + " in state=" + oldData.getEventType() + " was on deadserver; forcing offline"); ZKAssign.createOrForceNodeOffline(this.watcher, hri, this.master.getServerName()); @@ -958,12 +959,9 @@ public class AssignmentManager extends ZooKeeperListener { } @Override - public void nodeDeleted(String path) { - // Added so we notice when ephemeral nodes go away; in particular, - // SPLITTING or SPLIT nodes added by a regionserver splitting. + public void nodeDeleted(final String path) { if (path.startsWith(this.watcher.assignmentZNode)) { - String regionName = - ZKAssign.getRegionName(this.master.getZooKeeper(), path); + String regionName = ZKAssign.getRegionName(this.master.getZooKeeper(), path); RegionState rs = this.regionsInTransition.get(regionName); if (rs != null) { HRegionInfo regionInfo = rs.getRegion(); @@ -981,7 +979,7 @@ public class AssignmentManager extends ZooKeeperListener { } } } - + private void makeRegionOnline(RegionState rs, HRegionInfo regionInfo) { regionOnline(regionInfo, rs.serverName); LOG.info("The master has opened the region " @@ -989,14 +987,13 @@ public class AssignmentManager extends ZooKeeperListener { + rs.serverName); if (this.getZKTable().isDisablingOrDisabledTable( regionInfo.getTableNameAsString())) { - debugLog(regionInfo, "Opened region " + LOG.debug("Opened region " + regionInfo.getRegionNameAsString() + " but " + "this table is disabled, triggering close of region"); unassign(regionInfo); } } - /** * New unassigned node has been created. * @@ -1030,7 +1027,7 @@ public class AssignmentManager extends ZooKeeperListener { * @param regionInfo * @param sn */ - public void regionOnline(HRegionInfo regionInfo, ServerName sn) { + void regionOnline(HRegionInfo regionInfo, ServerName sn) { synchronized (this.regionsInTransition) { RegionState rs = this.regionsInTransition.remove(regionInfo.getEncodedName()); @@ -1186,7 +1183,7 @@ public class AssignmentManager extends ZooKeeperListener { */ public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan, boolean hijack) { - //If hijack is true do not call disableRegionIfInRIT as + // If hijack is true do not call disableRegionIfInRIT as // we have not yet moved the znode to OFFLINE state. if (!hijack && isDisabledorDisablingRegionInRIT(region)) { return; @@ -1441,13 +1438,11 @@ public class AssignmentManager extends ZooKeeperListener { } RegionPlan plan = getRegionPlan(state, forceNewPlan); if (plan == null) { - debugLog(state.getRegion(), - "Unable to determine a plan to assign " + state); + LOG.debug("Unable to determine a plan to assign " + state); return; // Should get reassigned later when RIT times out. } try { - debugLog(state.getRegion(), - "Assigning region " + state.getRegion().getRegionNameAsString() + + LOG.debug("Assigning region " + state.getRegion().getRegionNameAsString() + " to " + plan.getDestination().toString()); // Transition RegionState to PENDING_OPEN state.update(RegionState.State.PENDING_OPEN, System.currentTimeMillis(), @@ -1460,8 +1455,7 @@ public class AssignmentManager extends ZooKeeperListener { // Remove region from in-memory transition and unassigned node from ZK // While trying to enable the table the regions of the table were // already enabled. - debugLog(state.getRegion(), - "ALREADY_OPENED region " + state.getRegion().getRegionNameAsString() + + LOG.debug("ALREADY_OPENED region " + state.getRegion().getRegionNameAsString() + " to " + plan.getDestination().toString()); String encodedRegionName = state.getRegion() .getEncodedName(); @@ -1513,14 +1507,6 @@ public class AssignmentManager extends ZooKeeperListener { } } - private void debugLog(HRegionInfo region, String string) { - if (region.isMetaTable() || region.isRootRegion()) { - LOG.info(string); - } else { - LOG.debug(string); - } - } - private boolean isDisabledorDisablingRegionInRIT(final HRegionInfo region) { String tableName = region.getTableNameAsString(); boolean disabled = this.zkTable.isDisabledTable(tableName); @@ -1675,7 +1661,7 @@ public class AssignmentManager extends ZooKeeperListener { } if (newPlan) { - debugLog(state.getRegion(), "No previous transition plan was found (or we are ignoring " + + LOG.debug("No previous transition plan was found (or we are ignoring " + "an existing plan) for " + state.getRegion().getRegionNameAsString() + " so generated a random one; " + randomPlan + "; " + serverManager.countOfRegionServers() + @@ -1683,7 +1669,7 @@ public class AssignmentManager extends ZooKeeperListener { ", exclude=" + drainingServers.size() + ") available servers"); return randomPlan; } - debugLog(state.getRegion(), "Using pre-existing plan for region " + + LOG.debug("Using pre-existing plan for region " + state.getRegion().getRegionNameAsString() + "; plan=" + existingPlan); return existingPlan; } @@ -1722,7 +1708,11 @@ public class AssignmentManager extends ZooKeeperListener { /** * Unassigns the specified region. *
- * Updates the RegionState and sends the CLOSE RPC. + * Updates the RegionState and sends the CLOSE RPC unless region is being + * split by regionserver; then the unassign fails (silently) because we + * presume the region being unassigned no longer exists (its been split out + * of existence). TODO: What to do if split fails and is rolled back and + * parent is revivified? *
* If a RegionPlan is already set, it will remain. * @@ -1735,7 +1725,11 @@ public class AssignmentManager extends ZooKeeperListener { /** * Unassigns the specified region. *
- * Updates the RegionState and sends the CLOSE RPC. + * Updates the RegionState and sends the CLOSE RPC unless region is being + * split by regionserver; then the unassign fails (silently) because we + * presume the region being unassigned no longer exists (its been split out + * of existence). TODO: What to do if split fails and is rolled back and + * parent is revivified? *
* If a RegionPlan is already set, it will remain. * @@ -1743,12 +1737,13 @@ public class AssignmentManager extends ZooKeeperListener { * @param force if region should be closed even if already closing */ public void unassign(HRegionInfo region, boolean force) { - debugLog(region, "Starting unassignment of region " + + // TODO: Method needs refactoring. Ugly buried returns throughout. Beware! + LOG.debug("Starting unassignment of region " + region.getRegionNameAsString() + " (offlining)"); synchronized (this.regions) { // Check if this region is currently assigned if (!regions.containsKey(region)) { - debugLog(region, "Attempted to unassign region " + + LOG.debug("Attempted to unassign region " + region.getRegionNameAsString() + " but it is not " + "currently assigned anywhere"); return; @@ -1760,27 +1755,49 @@ public class AssignmentManager extends ZooKeeperListener { synchronized (regionsInTransition) { state = regionsInTransition.get(encodedName); if (state == null) { - // Create the znode in CLOSING state try { ZKAssign.createNodeClosing( master.getZooKeeper(), region, master.getServerName()); } catch (KeeperException e) { + if (e instanceof NodeExistsException) { + // Handle race between master initiated close and regionserver + // orchestrated splitting. See if existing node is in a + // SPLITTING or SPLIT state. If so, the regionserver started + // an op on node before we could get our CLOSING in. Deal. + NodeExistsException nee = (NodeExistsException)e; + String path = nee.getPath(); + try { + if (isSplitOrSplitting(path)) { + LOG.debug(path + " is SPLIT or SPLITTING; " + + "skipping unassign because region no longer exists -- its split"); + return; + } + } catch (KeeperException.NoNodeException ke) { + LOG.warn("Failed getData on SPLITTING/SPLIT at" + path + + "; presuming split and that the region to unassign, " + + encodedName + ", no longer exists -- confirm", ke); + return; + } catch (KeeperException ke) { + LOG.error("Unexpected zk state", ke); + ke = e; + } + } + // If we get here, don't understand whats going on -- abort. master.abort("Unexpected ZK exception creating node CLOSING", e); return; } state = new RegionState(region, RegionState.State.PENDING_CLOSE); regionsInTransition.put(encodedName, state); } else if (force && (state.isPendingClose() || state.isClosing())) { - debugLog(region, - "Attempting to unassign region " + region.getRegionNameAsString() + - " which is already " + state.getState() + - " but forcing to send a CLOSE RPC again "); + LOG.debug("Attempting to unassign region " + region.getRegionNameAsString() + + " which is already " + state.getState() + + " but forcing to send a CLOSE RPC again "); state.update(state.getState()); } else { - debugLog(region, "Attempting to unassign region " + + LOG.debug("Attempting to unassign region " + region.getRegionNameAsString() + " but it is " + - "already in transition (" + state.getState() + ")"); + "already in transition (" + state.getState() + ", force=" + force + ")"); return; } } @@ -1793,7 +1810,7 @@ public class AssignmentManager extends ZooKeeperListener { // TODO: We should consider making this look more like it does for the // region open where we catch all throwables and never abort if (serverManager.sendRegionClose(server, state.getRegion())) { - debugLog(region, "Sent CLOSE to " + server + " for region " + + LOG.debug("Sent CLOSE to " + server + " for region " + region.getRegionNameAsString()); return; } @@ -1827,7 +1844,7 @@ public class AssignmentManager extends ZooKeeperListener { } // RS is already processing this region, only need to update the timestamp if (t instanceof RegionAlreadyInTransitionException) { - debugLog(region, "update " + state + " the timestamp."); + LOG.debug("update " + state + " the timestamp."); state.update(state.getState()); } } @@ -1838,6 +1855,28 @@ public class AssignmentManager extends ZooKeeperListener { } /** + * @param path + * @return True if znode is in SPLIT or SPLITTING state. + * @throws KeeperException Can happen if the znode went away in meantime. + */ + private boolean isSplitOrSplitting(final String path) throws KeeperException { + boolean result = false; + // This may fail if the SPLIT or SPLITTING znode gets cleaned up before we + // can get data from it. + RegionTransitionData data = ZKAssign.getData(master.getZooKeeper(), path); + EventType evt = data.getEventType(); + switch (evt) { + case RS_ZK_REGION_SPLIT: + case RS_ZK_REGION_SPLITTING: + result = true; + break; + default: + break; + } + return result; + } + + /** * Waits until the specified region has completed assignment. *
* If the region is already assigned, returns immediately. Otherwise, method
@@ -2939,6 +2978,8 @@ public class AssignmentManager extends ZooKeeperListener {
/**
* Check whether the RegionServer is online.
+ * @param serverName
+ * @return True if online.
*/
public boolean isServerOnline(ServerName serverName) {
return this.serverManager.isServerOnline(serverName);
diff --git src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
index 0cc2f63..71e8445 100644
--- src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
+++ src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
@@ -818,6 +818,7 @@ public class SplitTransaction {
* @param region region to be created as offline
* @param serverName server event originates from
* @return Version of znode created.
+ * @throws KeeperException
* @throws IOException
*/
private static int createNodeSplitting(final ZooKeeperWatcher zkw,
diff --git src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
index 4930f66..4f8aea2 100644
--- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
+++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java
@@ -81,8 +81,6 @@ public class ZKTable {
/**
* Gets a list of all the tables set as disabled in zookeeper.
- * @param zkw
- * @return list of disabled tables, empty list if none
* @throws KeeperException
*/
private void populateTableStates()
@@ -90,6 +88,7 @@ public class ZKTable {
synchronized (this.cache) {
List Does not transition nodes from other states. If a node already exists
+ * for this region, a {@link NodeExistsException} will be thrown.
+ *
+ * @param zkw zk reference
+ * @param region region to be created as offline
+ * @param serverName server event originates from
+ * @return Version of znode created.
+ * @throws KeeperException
+ * @throws IOException
+ */
+ // Copied from SplitTransaction rather than open the method over there in
+ // the regionserver package.
+ private static int createNodeSplitting(final ZooKeeperWatcher zkw,
+ final HRegionInfo region, final ServerName serverName)
+ throws KeeperException, IOException {
+ RegionTransitionData data =
+ new RegionTransitionData(EventType.RS_ZK_REGION_SPLITTING,
+ region.getRegionName(), serverName);
+
+ String node = ZKAssign.getNodeName(zkw, region.getEncodedName());
+ if (!ZKUtil.createEphemeralNodeAndWatch(zkw, node, data.getBytes())) {
+ throw new IOException("Failed create of ephemeral " + node);
+ }
+ // Transition node from SPLITTING to SPLITTING and pick up version so we
+ // can be sure this znode is ours; version is needed deleting.
+ return transitionNodeSplitting(zkw, region, serverName, -1);
+ }
+
+ // Copied from SplitTransaction rather than open the method over there in
+ // the regionserver package.
+ private static int transitionNodeSplitting(final ZooKeeperWatcher zkw,
+ final HRegionInfo parent,
+ final ServerName serverName, final int version)
+ throws KeeperException, IOException {
+ return ZKAssign.transitionNode(zkw, parent, serverName,
+ EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version);
+ }
+
+ private void unassign(final AssignmentManager am, final ServerName sn,
+ final HRegionInfo hri) {
+ // Before I can unassign a region, I need to set it online.
+ am.regionOnline(hri, sn);
+ // Unassign region.
+ am.unassign(hri);
+ }
+}
\ No newline at end of file