Index: src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (revision 1136562) +++ src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (working copy) @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -160,7 +161,14 @@ Thread.sleep(1 * 1000); ArrayList servers = new ArrayList(); servers.add(rsAddressOrig); - HBaseFsckRepair.fixDupeAssignment(conf, hriOrig, servers); + try { + HBaseFsckRepair.fixDupeAssignment(TEST_UTIL.getHBaseAdmin(), hriOrig, servers); + } catch (IOException ex) { + ex = RemoteExceptionHandler.checkIOException(ex); + if (!(ex instanceof UnknownRegionException)) { + fail("Unexpected exception: " + ex); + } + } // We created 1 table, should be fine assertNoErrors(doFsck(false)); Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java (working copy) @@ -20,9 +20,7 @@ package org.apache.hadoop.hbase.zookeeper; import java.io.IOException; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.logging.Log; @@ -68,9 +66,6 @@ private final List listeners = new CopyOnWriteArrayList(); - // set of unassigned nodes watched - private Set unassignedNodes = new HashSet(); - // node names // base znode for this cluster @@ -363,14 +358,6 @@ } /** - * Get the set of already watched unassigned nodes. - * @return Set of Nodes. - */ - public Set getNodes() { - return unassignedNodes; - } - - /** * Handles KeeperExceptions in client calls. * * This may be temporary but for now this gives one place to deal with these. Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java (working copy) @@ -396,30 +396,6 @@ } /** - * Atomically add watches and read data from all unwatched unassigned nodes. - * - *

This works because master is the only person deleting nodes. - */ - public static List watchAndGetNewChildren(ZooKeeperWatcher zkw, - String baseNode) - throws KeeperException { - List newNodes = new ArrayList(); - synchronized(zkw.getNodes()) { - List nodes = - ZKUtil.listChildrenAndWatchForNewChildren(zkw, baseNode); - for(String node : nodes) { - String nodePath = ZKUtil.joinZNode(baseNode, node); - if(!zkw.getNodes().contains(nodePath)) { - byte [] data = ZKUtil.getDataAndWatch(zkw, nodePath); - newNodes.add(new NodeAndData(nodePath, data)); - zkw.getNodes().add(nodePath); - } - } - } - return newNodes; - } - - /** * Simple class to hold a node path and node data. */ public static class NodeAndData { Index: src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/RegionServerTracker.java (working copy) @@ -61,7 +61,7 @@ */ public void start() throws KeeperException { watcher.registerListener(this); - ZKUtil.watchAndGetNewChildren(watcher, watcher.rsZNode); + ZKUtil.listChildrenAndWatchThem(watcher, watcher.rsZNode); } @Override @@ -83,7 +83,7 @@ public void nodeChildrenChanged(String path) { if(path.equals(watcher.rsZNode)) { try { - ZKUtil.watchAndGetNewChildren(watcher, watcher.rsZNode); + ZKUtil.listChildrenAndWatchThem(watcher, watcher.rsZNode); } catch (KeeperException e) { abortable.abort("Unexpected zk exception getting RS nodes", e); } Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java (working copy) @@ -142,11 +142,8 @@ region.getEncodedName() + " in OFFLINE state")); RegionTransitionData data = new RegionTransitionData(event, region.getRegionName(), serverName); - synchronized(zkw.getNodes()) { - String node = getNodeName(zkw, region.getEncodedName()); - zkw.getNodes().add(node); - ZKUtil.createAndWatch(zkw, node, data.getBytes()); - } + String node = getNodeName(zkw, region.getEncodedName()); + ZKUtil.createAndWatch(zkw, node, data.getBytes()); } /** @@ -172,11 +169,8 @@ region.getEncodedName() + " with OFFLINE state")); RegionTransitionData data = new RegionTransitionData( EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName); - synchronized(zkw.getNodes()) { - String node = getNodeName(zkw, region.getEncodedName()); - zkw.getNodes().add(node); - ZKUtil.asyncCreate(zkw, node, data.getBytes(), cb, ctx); - } + String node = getNodeName(zkw, region.getEncodedName()); + ZKUtil.asyncCreate(zkw, node, data.getBytes(), cb, ctx); } /** @@ -204,11 +198,8 @@ region.getEncodedName() + " to OFFLINE state")); RegionTransitionData data = new RegionTransitionData( EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName); - synchronized(zkw.getNodes()) { - String node = getNodeName(zkw, region.getEncodedName()); - zkw.getNodes().add(node); - ZKUtil.setData(zkw, node, data.getBytes()); - } + String node = getNodeName(zkw, region.getEncodedName()); + ZKUtil.setData(zkw, node, data.getBytes()); } @@ -237,25 +228,22 @@ region.getEncodedName() + " with OFFLINE state")); RegionTransitionData data = new RegionTransitionData( EventType.M_ZK_REGION_OFFLINE, region.getRegionName(), serverName); - synchronized(zkw.getNodes()) { - String node = getNodeName(zkw, region.getEncodedName()); - zkw.sync(node); - zkw.getNodes().add(node); - int version = ZKUtil.checkExists(zkw, node); - if(version == -1) { - ZKUtil.createAndWatch(zkw, node, data.getBytes()); + String node = getNodeName(zkw, region.getEncodedName()); + zkw.sync(node); + int version = ZKUtil.checkExists(zkw, node); + if(version == -1) { + ZKUtil.createAndWatch(zkw, node, data.getBytes()); + } else { + if (!ZKUtil.setData(zkw, node, data.getBytes(), version)) { + return false; } else { - if (!ZKUtil.setData(zkw, node, data.getBytes(), version)) { - return false; - } else { - // We successfully forced to OFFLINE, reset watch and handle if - // the state changed in between our set and the watch - RegionTransitionData curData = + // We successfully forced to OFFLINE, reset watch and handle if + // the state changed in between our set and the watch + RegionTransitionData curData = ZKAssign.getData(zkw, region.getEncodedName()); - if (curData.getEventType() != data.getEventType()) { - // state changed, need to process - return false; - } + if (curData.getEventType() != data.getEventType()) { + // state changed, need to process + return false; } } } @@ -407,20 +395,16 @@ " state but node is in " + data.getEventType() + " state")); return false; } - synchronized(zkw.getNodes()) { - // TODO: Does this go here or only if we successfully delete node? - zkw.getNodes().remove(node); - if(!ZKUtil.deleteNode(zkw, node, stat.getVersion())) { - LOG.warn(zkw.prefix("Attempting to delete " + + if(!ZKUtil.deleteNode(zkw, node, stat.getVersion())) { + LOG.warn(zkw.prefix("Attempting to delete " + "unassigned node in " + expectedState + - " state but " + - "after verifying it was in OPENED state, we got a version mismatch")); - return false; - } - LOG.debug(zkw.prefix("Successfully deleted unassigned node for region " + - regionName + " in expected state " + expectedState)); - return true; + " state but " + + "after verifying it was in OPENED state, we got a version mismatch")); + return false; } + LOG.debug(zkw.prefix("Successfully deleted unassigned node for region " + + regionName + " in expected state " + expectedState)); + return true; } /** @@ -473,11 +457,8 @@ RegionTransitionData data = new RegionTransitionData( EventType.RS_ZK_REGION_CLOSING, region.getRegionName(), serverName); - synchronized (zkw.getNodes()) { - String node = getNodeName(zkw, region.getEncodedName()); - zkw.getNodes().add(node); - return ZKUtil.createAndWatch(zkw, node, data.getBytes()); - } + String node = getNodeName(zkw, region.getEncodedName()); + return ZKUtil.createAndWatch(zkw, node, data.getBytes()); } /** @@ -775,6 +756,19 @@ } /** + * Get the version of the specified znode + * @param zkw zk reference + * @param region region's info + * @return the version of the znode, -1 if it doesn't exist + * @throws KeeperException + */ + public static int getVersion(ZooKeeperWatcher zkw, HRegionInfo region) + throws KeeperException { + String znode = getNodeName(zkw, region.getEncodedName()); + return ZKUtil.checkExists(zkw, znode); + } + + /** * Delete the assignment node regardless of its current state. *

* Fail silent even if the node does not exist at all. Index: src/main/java/org/apache/hadoop/hbase/HConstants.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/HConstants.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/HConstants.java (working copy) @@ -358,9 +358,6 @@ public static final String REPLICATION_ENABLE_KEY = "hbase.replication"; - /** HBCK special code name used as server name when manipulating ZK nodes */ - public static final String HBCK_CODE_NAME = "HBCKServerName"; - public static final String HBASE_MASTER_LOGCLEANER_PLUGINS = "hbase.master.logcleaner.plugins"; Index: src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java (working copy) @@ -109,7 +109,7 @@ int expectedVersion = FAILED; if (this.zk) { - expectedVersion = setClosingState(); + expectedVersion = getCurrentVersion(); if (expectedVersion == FAILED) return; } @@ -169,16 +169,16 @@ } /** - * Create ZK node in CLOSING state. - * @return The expectedVersion. If -1, we failed setting CLOSING. + * Get the node's current version + * @return The expectedVersion. If -1, we failed getting the node */ - private int setClosingState() { + private int getCurrentVersion() { int expectedVersion = FAILED; try { - if ((expectedVersion = ZKAssign.createNodeClosing( - server.getZooKeeper(), regionInfo, server.getServerName())) == FAILED) { - LOG.warn("Error creating node in CLOSING state, aborting close of " + - regionInfo.getRegionNameAsString()); + if ((expectedVersion = ZKAssign.getVersion( + server.getZooKeeper(), regionInfo)) == FAILED) { + LOG.warn("Error getting node's version in CLOSING state," + + " aborting close of " + regionInfo.getRegionNameAsString()); } } catch (KeeperException e) { LOG.warn("Error creating node in CLOSING state, aborting close of " + Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -38,6 +38,7 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -354,11 +355,6 @@ LOG.warn("Unexpected NULL input " + data); return; } - // Check if this is a special HBCK transition - if (data.getServerName().equals(HConstants.HBCK_CODE_NAME)) { - handleHBCK(data); - return; - } // Verify this is a known server if (!serverManager.isServerOnline(data.getServerName()) && !this.master.getServerName().equals(data.getServerName())) { @@ -451,45 +447,6 @@ } } - /** - * Handle a ZK unassigned node transition triggered by HBCK repair tool. - *

- * This is handled in a separate code path because it breaks the normal rules. - * @param data - */ - private void handleHBCK(RegionTransitionData data) { - String encodedName = HRegionInfo.encodeRegionName(data.getRegionName()); - LOG.info("Handling HBCK triggered transition=" + data.getEventType() + - ", server=" + data.getServerName() + ", region=" + - HRegionInfo.prettyPrint(encodedName)); - RegionState regionState = regionsInTransition.get(encodedName); - switch (data.getEventType()) { - case M_ZK_REGION_OFFLINE: - HRegionInfo regionInfo = null; - if (regionState != null) { - regionInfo = regionState.getRegion(); - } else { - try { - regionInfo = MetaReader.getRegion(catalogTracker, - data.getRegionName()).getFirst(); - } catch (IOException e) { - LOG.info("Exception reading META doing HBCK repair operation", e); - return; - } - } - LOG.info("HBCK repair is triggering assignment of region=" + - regionInfo.getRegionNameAsString()); - // trigger assign, node is already in OFFLINE so don't need to update ZK - assign(regionInfo, false); - break; - - default: - LOG.warn("Received unexpected region state from HBCK (" + - data.getEventType() + ")"); - break; - } - } - // ZooKeeper events /** @@ -507,16 +464,14 @@ @Override public void nodeCreated(String path) { if(path.startsWith(watcher.assignmentZNode)) { - synchronized(regionsInTransition) { - try { - RegionTransitionData data = ZKAssign.getData(watcher, path); - if(data == null) { - return; - } - handleRegion(data); - } catch (KeeperException e) { - master.abort("Unexpected ZK exception reading unassigned node data", e); + try { + RegionTransitionData data = ZKAssign.getData(watcher, path); + if(data == null) { + return; } + handleRegion(data); + } catch (KeeperException e) { + master.abort("Unexpected ZK exception reading unassigned node data", e); } } } @@ -536,16 +491,14 @@ @Override public void nodeDataChanged(String path) { if(path.startsWith(watcher.assignmentZNode)) { - synchronized(regionsInTransition) { - try { - RegionTransitionData data = ZKAssign.getData(watcher, path); - if(data == null) { - return; - } - handleRegion(data); - } catch (KeeperException e) { - master.abort("Unexpected ZK exception reading unassigned node data", e); + try { + RegionTransitionData data = ZKAssign.getData(watcher, path); + if(data == null) { + return; } + handleRegion(data); + } catch (KeeperException e) { + master.abort("Unexpected ZK exception reading unassigned node data", e); } } } @@ -560,23 +513,17 @@ *

    *
  1. Watch the node for further children changed events
  2. *
  3. Watch all new children for changed events
  4. - *
  5. Read all children and handle them
  6. *
*/ @Override public void nodeChildrenChanged(String path) { if(path.equals(watcher.assignmentZNode)) { - synchronized(regionsInTransition) { - try { - List newNodes = ZKUtil.watchAndGetNewChildren(watcher, - watcher.assignmentZNode); - for(NodeAndData newNode : newNodes) { - LOG.debug("Handling new unassigned node: " + newNode); - handleRegion(RegionTransitionData.fromBytes(newNode.getData())); - } - } catch(KeeperException e) { - master.abort("Unexpected ZK exception reading unassigned children", e); - } + try { + // Just make sure we see the changes for the new znodes + ZKUtil.listChildrenAndWatchThem(watcher, + watcher.assignmentZNode); + } catch(KeeperException e) { + master.abort("Unexpected ZK exception reading unassigned children", e); } } } @@ -645,9 +592,7 @@ rs = this.regionsInTransition.get(e.getKey()); } if (rs == null) continue; - synchronized (rs) { - rs.update(rs.getState()); - } + rs.setStamp(); } } @@ -1102,9 +1047,20 @@ synchronized (regionsInTransition) { state = regionsInTransition.get(encodedName); if (state == null) { + + // Create the znode in CLOSING state + try { + ZKAssign.createNodeClosing( + master.getZooKeeper(), region, master.getServerName()); + } catch (KeeperException e) { + master.abort("Unexpected ZK exception creating node CLOSING", e); + return; + } state = new RegionState(region, RegionState.State.PENDING_CLOSE); regionsInTransition.put(encodedName, state); } else if (force && state.isPendingClose()) { + // JD 05/25/11 + // in my experience this is useless, when this happens it just spins LOG.debug("Attempting to unassign region " + region.getRegionNameAsString() + " which is already pending close " + "but forcing an additional close"); @@ -1663,9 +1619,7 @@ " has been CLOSED for too long, waiting on queued " + "ClosedRegionHandler to run or server shutdown"); // Update our timestamp. - synchronized(regionState) { - regionState.update(regionState.getState()); - } + regionState.setStamp(); break; case OFFLINE: LOG.info("Region has been OFFLINE for too long, " + @@ -1968,9 +1922,12 @@ } private State state; - private long stamp; + // Many threads can update the state at the stamp at the same time + private final AtomicLong stamp; - public RegionState() {} + public RegionState() { + this.stamp = new AtomicLong(System.currentTimeMillis()); + } RegionState(HRegionInfo region, State state) { this(region, state, System.currentTimeMillis()); @@ -1979,25 +1936,33 @@ RegionState(HRegionInfo region, State state, long stamp) { this.region = region; this.state = state; - this.stamp = stamp; + this.stamp = new AtomicLong(stamp); } public void update(State state, long stamp) { this.state = state; - this.stamp = stamp; + setStamp(stamp); } public void update(State state) { this.state = state; - this.stamp = System.currentTimeMillis(); + setStamp(); } + public void setStamp(long stamp) { + this.stamp.set(stamp); + } + + public void setStamp() { + this.stamp.set(System.currentTimeMillis()); + } + public State getState() { return state; } public long getStamp() { - return stamp; + return stamp.get(); } public HRegionInfo getRegion() { @@ -2043,14 +2008,14 @@ region = new HRegionInfo(); region.readFields(in); state = State.valueOf(in.readUTF()); - stamp = in.readLong(); + stamp.set(in.readLong()); } @Override public void write(DataOutput out) throws IOException { region.write(out); out.writeUTF(state.name()); - out.writeLong(stamp); + out.writeLong(stamp.get()); } } Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy) @@ -91,6 +91,7 @@ // Empty regioninfo qualifiers in .META. private Set emptyRegionInfoQualifiers = new HashSet(); private int numThreads = MAX_NUM_THREADS; + private final HBaseAdmin admin; ThreadPoolExecutor executor; // threads to retrieve data from regionservers @@ -105,7 +106,7 @@ throws MasterNotRunningException, ZooKeeperConnectionException, IOException { this.conf = conf; - HBaseAdmin admin = new HBaseAdmin(conf); + admin = new HBaseAdmin(conf); status = admin.getMaster().getClusterStatus(); connection = admin.getConnection(); @@ -425,7 +426,7 @@ if (shouldFix()) { errors.print("Trying to fix unassigned region..."); setShouldRerun(); - HBaseFsckRepair.fixUnassigned(this.conf, hbi.metaEntry); + HBaseFsckRepair.fixUnassigned(this.admin, hbi.metaEntry); } } else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) { errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, "Region " @@ -440,7 +441,7 @@ if (shouldFix()) { errors.print("Trying to fix assignment error..."); setShouldRerun(); - HBaseFsckRepair.fixDupeAssignment(this.conf, hbi.metaEntry, hbi.deployedOn); + HBaseFsckRepair.fixDupeAssignment(this.admin, hbi.metaEntry, hbi.deployedOn); } } else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) { errors.reportError(ERROR_CODE.SERVER_DOES_NOT_MATCH_META, "Region " @@ -451,7 +452,7 @@ if (shouldFix()) { errors.print("Trying to fix assignment error..."); setShouldRerun(); - HBaseFsckRepair.fixDupeAssignment(this.conf, hbi.metaEntry, hbi.deployedOn); + HBaseFsckRepair.fixDupeAssignment(this.admin, hbi.metaEntry, hbi.deployedOn); } } else { errors.reportError(ERROR_CODE.UNKNOWN, "Region " + descriptiveName + @@ -683,7 +684,7 @@ errors.print("Trying to fix a problem with .META..."); setShouldRerun(); // try to fix it (treat it as unassigned region) - HBaseFsckRepair.fixUnassigned(conf, root.metaEntry); + HBaseFsckRepair.fixUnassigned(admin, root.metaEntry); } } // If there are more than one regions pretending to hold the .META. @@ -697,7 +698,7 @@ for (HbckInfo mRegion : metaRegions) { deployedOn.add(mRegion.metaEntry.regionServer); } - HBaseFsckRepair.fixDupeAssignment(conf, root.metaEntry, deployedOn); + HBaseFsckRepair.fixDupeAssignment(admin, root.metaEntry, deployedOn); } } // rerun hbck with hopefully fixed META Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (revision 1136562) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (working copy) @@ -23,14 +23,13 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.ipc.HRegionInterface; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.zookeeper.KeeperException; public class HBaseFsckRepair { @@ -39,14 +38,14 @@ * Fix dupe assignment by doing silent closes on each RS hosting the region * and then force ZK unassigned node to OFFLINE to trigger assignment by * master. - * @param conf + * @param admin * @param region * @param servers * @throws IOException * @throws KeeperException * @throws InterruptedException */ - public static void fixDupeAssignment(Configuration conf, HRegionInfo region, + public static void fixDupeAssignment(HBaseAdmin admin, HRegionInfo region, List servers) throws IOException, KeeperException, InterruptedException { @@ -54,35 +53,31 @@ // Close region on the servers silently for(HServerAddress server : servers) { - closeRegionSilentlyAndWait(conf, server, actualRegion); + closeRegionSilentlyAndWait(admin.getConfiguration(), server, actualRegion); } // Force ZK node to OFFLINE so master assigns - forceOfflineInZK(conf, actualRegion); + forceOfflineInZK(admin, actualRegion); } /** - * Fix unassigned by creating/transition the unassigned ZK node for this - * region to OFFLINE state with a special flag to tell the master that this - * is a forced operation by HBCK. - * @param conf + * Ask the master to force reassign the region + * @param admin * @param region * @throws IOException * @throws KeeperException */ - public static void fixUnassigned(Configuration conf, HRegionInfo region) + public static void fixUnassigned(HBaseAdmin admin, HRegionInfo region) throws IOException, KeeperException { HRegionInfo actualRegion = new HRegionInfo(region); // Force ZK node to OFFLINE so master assigns - forceOfflineInZK(conf, actualRegion); + forceOfflineInZK(admin, actualRegion); } - private static void forceOfflineInZK(Configuration conf, HRegionInfo region) + private static void forceOfflineInZK(HBaseAdmin admin, HRegionInfo region) throws ZooKeeperConnectionException, KeeperException, IOException { - ZKAssign.createOrForceNodeOffline( - HConnectionManager.getConnection(conf).getZooKeeperWatcher(), - region, HConstants.HBCK_CODE_NAME); + admin.assign(region.getRegionName(), true); } protected static void closeRegionSilentlyAndWait(Configuration conf,