Index: src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (revision 1048879) +++ src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (working copy) @@ -19,7 +19,7 @@ */ package org.apache.hadoop.hbase.util; -import java.io.IOException; +import static org.junit.Assert.assertEquals; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,8 +36,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertEquals; - public class TestHBaseFsck { final Log LOG = LogFactory.getLog(getClass()); @@ -53,7 +51,7 @@ } @Test - public void testHBaseFsck() throws IOException { + public void testHBaseFsck() throws Exception { HBaseFsck fsck = new HBaseFsck(conf); fsck.displayFullReport(); fsck.setTimeLag(0); @@ -71,7 +69,7 @@ // point to a different region server HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); ResultScanner scanner = meta.getScanner(new Scan()); - + resforloop : for (Result res : scanner) { long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER)); @@ -100,6 +98,7 @@ // Fixed or not, it still reports inconsistencies assertEquals(-1, result); + Thread.sleep(15000); // Disabled, won't work because the region stays unassigned, see HBASE-3217 // new HTable(conf, TABLE).getScanner(new Scan()); } Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (revision 1048879) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (working copy) @@ -313,4 +313,39 @@ } return disabledTables; } + + /** + * Gets a list of all the tables set as disabled in zookeeper. + * @return Set of disabled tables, empty Set if none + * @throws KeeperException + */ + public static Set getDisabledTables(ZooKeeperWatcher zkw) + throws KeeperException { + Set disabledTables = new HashSet(); + List children = + ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode); + for (String child: children) { + TableState state = getTableState(zkw, child); + if (state == TableState.DISABLED) disabledTables.add(child); + } + return disabledTables; + } + + /** + * Gets a list of all the tables set as disabled in zookeeper. + * @return Set of disabled tables, empty Set if none + * @throws KeeperException + */ + public static Set getDisabledOrDisablingTables(ZooKeeperWatcher zkw) + throws KeeperException { + Set disabledTables = new HashSet(); + List children = + ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode); + for (String child: children) { + TableState state = getTableState(zkw, child); + if (state == TableState.DISABLED || state == TableState.DISABLING) + disabledTables.add(child); + } + return disabledTables; + } } \ No newline at end of file Index: src/main/java/org/apache/hadoop/hbase/HConstants.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/HConstants.java (revision 1048879) +++ src/main/java/org/apache/hadoop/hbase/HConstants.java (working copy) @@ -358,6 +358,9 @@ public static final String REPLICATION_ENABLE_KEY = "hbase.replication"; + /** HBCK special code name used as server name when manipulating ZK nodes */ + public static final String HBCK_CODE_NAME = "HBCKsecret123"; + private HConstants() { // Can't be instantiated with this ctor. } Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1048879) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -40,6 +40,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Chore; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; @@ -51,10 +52,9 @@ import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.catalog.RootLocationEditor; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.RegionTransitionData; -import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; +import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan; import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler; import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler; @@ -65,9 +65,9 @@ import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTable; import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData; import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData; import org.apache.hadoop.io.Writable; import org.apache.hadoop.ipc.RemoteException; import org.apache.zookeeper.AsyncCallback; @@ -143,7 +143,7 @@ * @param serverManager * @param catalogTracker * @param service - * @throws KeeperException + * @throws KeeperException */ public AssignmentManager(Server master, ServerManager serverManager, CatalogTracker catalogTracker, final ExecutorService service) @@ -337,6 +337,11 @@ LOG.warn("Unexpected NULL input " + data); return; } + // Check if this is a special HBCK transition + if (data.getServerName().equals(HConstants.HBCK_CODE_NAME)) { + handleHBCK(data); + return; + } // Verify this is a known server if (!serverManager.isServerOnline(data.getServerName()) && !this.master.getServerName().equals(data.getServerName())) { @@ -424,6 +429,45 @@ } } + /** + * Handle a ZK unassigned node transition triggered by HBCK repair tool. + *

+ * This is handled in a separate code path because it breaks the normal rules. + * @param data + */ + private void handleHBCK(RegionTransitionData data) { + String encodedName = HRegionInfo.encodeRegionName(data.getRegionName()); + LOG.info("Handling HBCK triggered transition=" + data.getEventType() + + ", server=" + data.getServerName() + ", region=" + + HRegionInfo.prettyPrint(encodedName)); + RegionState regionState = regionsInTransition.get(encodedName); + switch (data.getEventType()) { + case M_ZK_REGION_OFFLINE: + HRegionInfo regionInfo = null; + if (regionState != null) { + regionInfo = regionState.getRegion(); + } else { + try { + regionInfo = MetaReader.getRegion(catalogTracker, + data.getRegionName()).getFirst(); + } catch (IOException e) { + LOG.info("Exception reading META doing HBCK repair operation", e); + return; + } + } + LOG.info("HBCK repair is triggering assignment of region=" + + regionInfo.getRegionNameAsString()); + // trigger assign, node is already in OFFLINE so don't need to update ZK + assign(regionInfo, false); + break; + + default: + LOG.warn("Received unexpected region state from HBCK (" + + data.getEventType() + ")"); + break; + } + } + // ZooKeeper events /** @@ -1001,7 +1045,7 @@ public void unassign(HRegionInfo region, boolean force) { LOG.debug("Starting unassignment of region " + region.getRegionNameAsString() + " (offlining)"); - synchronized (this.regions) { + synchronized (this.regions) { // Check if this region is currently assigned if (!regions.containsKey(region)) { LOG.debug("Attempted to unassign region " + @@ -1610,9 +1654,9 @@ /** * Process shutdown server removing any assignments. * @param hsi Server that went down. - * @return set of regions on this server that are not in transition + * @return list of regions in transition on this server */ - public List processServerShutdown(final HServerInfo hsi) { + public List processServerShutdown(final HServerInfo hsi) { // Clean out any existing assignment plans for this server synchronized (this.regionPlans) { for (Iterator > i = @@ -1628,7 +1672,7 @@ // Remove this server from map of servers to regions, and remove all regions // of this server from online map of regions. Set deadRegions = null; - List rits = new ArrayList(); + List rits = new ArrayList(); synchronized (this.regions) { List assignedRegions = this.servers.remove(hsi); if (assignedRegions == null || assignedRegions.isEmpty()) { @@ -1646,7 +1690,7 @@ synchronized (regionsInTransition) { for (RegionState region : this.regionsInTransition.values()) { if (deadRegions.remove(region.getRegion())) { - rits.add(region.getRegion()); + rits.add(region); } } } Index: src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (revision 1048879) +++ src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (working copy) @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.DeadServer; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.AssignmentManager.RegionState; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Writables; import org.apache.zookeeper.KeeperException; @@ -98,7 +99,7 @@ // doing after log splitting. Could do some states before -- OPENING? // OFFLINE? -- and then others after like CLOSING that depend on log // splitting. - List regionsInTransition = + List regionsInTransition = this.services.getAssignmentManager().processServerShutdown(this.hsi); // Assign root and meta if we were carrying them. @@ -133,11 +134,16 @@ } } - // Remove regions that were in transition - for (HRegionInfo rit : regionsInTransition) hris.remove(rit); - LOG.info("Reassigning the " + hris.size() + " region(s) that " + serverName + // Skip regions that were in transition unless CLOSING or PENDING_CLOSE + for (RegionState rit : regionsInTransition) { + if (!rit.isClosing() && !rit.isPendingClose()) { + hris.remove(rit.getRegion()); + } + } + + LOG.info("Reassigning " + hris.size() + " region(s) that " + serverName + " was carrying (skipping " + regionsInTransition.size() + - " regions(s) that are in transition)"); + " regions(s) that are already in transition)"); // Iterate regions that were on this server and assign them for (Map.Entry e: hris.entrySet()) { Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1048879) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy) @@ -23,11 +23,11 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -49,19 +49,21 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; -import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.regionserver.wal.HLog; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; import com.google.common.base.Joiner; import com.google.common.collect.Lists; /** - * Check consistency among the in-memory states of the master and the + * Check consistency among the in-memory states of the master and the * region server(s) and the state of data in HDFS. */ public class HBaseFsck { @@ -75,6 +77,8 @@ private TreeMap regionInfo = new TreeMap(); private TreeMap tablesInfo = new TreeMap(); + private TreeSet disabledTables = + new TreeSet(Bytes.BYTES_COMPARATOR); ErrorReporter errors = new PrintingErrorReporter(); private static boolean details = false; // do we display the full report @@ -92,7 +96,7 @@ * @throws MasterNotRunningException if the master is not running * @throws ZooKeeperConnectionException if unable to connect to zookeeper */ - public HBaseFsck(Configuration conf) + public HBaseFsck(Configuration conf) throws MasterNotRunningException, ZooKeeperConnectionException, IOException { this.conf = conf; @@ -105,8 +109,10 @@ * Contacts the master and prints out cluster-wide information * @throws IOException if a remote or network exception occurs * @return 0 on success, non-zero on failure + * @throws KeeperException + * @throws InterruptedException */ - int doWork() throws IOException { + int doWork() throws IOException, KeeperException, InterruptedException { // print hbase server version errors.print("Version: " + status.getHBaseVersion()); @@ -114,8 +120,8 @@ regionInfo.clear(); tablesInfo.clear(); emptyRegionInfoQualifiers.clear(); + disabledTables.clear(); - // get a list of all regions from the master. This involves // scanning the META table if (!recordRootRegion()) { @@ -152,7 +158,7 @@ // From the master, get a list of all known live region servers Collection regionServers = status.getServerInfo(); - errors.print("Number of live region servers: " + + errors.print("Number of live region servers: " + regionServers.size()); if (details) { for (HServerInfo rsinfo: regionServers) { @@ -162,7 +168,7 @@ // From the master, get a list of all dead region servers Collection deadRegionServers = status.getDeadServerNames(); - errors.print("Number of dead region servers: " + + errors.print("Number of dead region servers: " + deadRegionServers.size()); if (details) { for (String name: deadRegionServers) { @@ -185,6 +191,9 @@ } } + // Get disabled tables from ZooKeeper + loadDisabledTables(); + // Check consistency checkConsistency(); @@ -198,6 +207,31 @@ } /** + * Load the list of disabled tables in ZK into local set. + * @throws ZooKeeperConnectionException + * @throws IOException + * @throws KeeperException + */ + private void loadDisabledTables() + throws ZooKeeperConnectionException, IOException, KeeperException { + ZooKeeperWatcher zkw = + HConnectionManager.getConnection(conf).getZooKeeperWatcher(); + for (String tableName : ZKTable.getDisabledOrDisablingTables(zkw)) { + disabledTables.add(Bytes.toBytes(tableName)); + } + } + + /** + * Check if the specified region's table is disabled. + * @throws ZooKeeperConnectionException + * @throws IOException + * @throws KeeperException + */ + private boolean isTableDisabled(HRegionInfo regionInfo) { + return disabledTables.contains(regionInfo.getTableDesc().getName()); + } + + /** * Scan HDFS for all regions, recording their information into * regionInfo */ @@ -280,7 +314,7 @@ regionInfo.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo); return true; } - + /** * Contacts each regionserver and fetches metadata about regions. * @param regionServerList - the list of region servers to connect to @@ -315,7 +349,7 @@ HbckInfo hbi = getOrCreateInfo(r.getEncodedName()); hbi.deployedOn.add(rsinfo.getServerAddress()); } - } catch (IOException e) { // unable to connect to the region server. + } catch (IOException e) { // unable to connect to the region server. errors.reportError("\nRegionServer:" + rsinfo.getServerName() + " Unable to fetch region information. " + e); } @@ -324,8 +358,11 @@ /** * Check consistency of all regions that have been found in previous phases. + * @throws KeeperException + * @throws InterruptedException */ - void checkConsistency() throws IOException { + void checkConsistency() + throws IOException, KeeperException, InterruptedException { for (java.util.Map.Entry e: regionInfo.entrySet()) { doConsistencyCheck(e.getKey(), e.getValue()); } @@ -333,9 +370,11 @@ /** * Check a single region for consistency and correct deployment. + * @throws KeeperException + * @throws InterruptedException */ void doConsistencyCheck(final String key, final HbckInfo hbi) - throws IOException { + throws IOException, KeeperException, InterruptedException { String descriptiveName = hbi.toString(); boolean inMeta = hbi.metaEntry != null; @@ -346,7 +385,7 @@ boolean deploymentMatchesMeta = hasMetaAssignment && isDeployed && !isMultiplyDeployed && hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0)); - boolean shouldBeDeployed = inMeta && !hbi.metaEntry.isOffline(); + boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry); boolean recentlyModified = hbi.foundRegionDir != null && hbi.foundRegionDir.getModificationTime() + timelag > System.currentTimeMillis(); @@ -533,8 +572,8 @@ /** * Return a list of user-space table names whose metadata have not been * modified in the last few milliseconds specified by timelag - * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER, - * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last + * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER, + * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last * milliseconds specified by timelag, then the table is a candidate to be returned. * @param regionList - all entries found in .META * @return tables that have not been modified recently @@ -580,8 +619,11 @@ * If there are inconsistencies (i.e. zero or more than one regions * pretend to be holding the .META.) try to fix that and report an error. * @throws IOException from HBaseFsckRepair functions + * @throws KeeperException + * @throws InterruptedException */ - boolean checkMetaEntries() throws IOException { + boolean checkMetaEntries() + throws IOException, KeeperException, InterruptedException { List metaRegions = Lists.newArrayList(); for (HbckInfo value : regionInfo.values()) { if (value.metaEntry.isMetaTable()) { @@ -709,7 +751,7 @@ HServerAddress regionServer; // server hosting this region long modTime; // timestamp of most recent modification metadata - public MetaEntry(HRegionInfo rinfo, HServerAddress regionServer, + public MetaEntry(HRegionInfo rinfo, HServerAddress regionServer, byte[] startCode, long modTime) { super(rinfo); this.regionServer = regionServer; @@ -883,16 +925,16 @@ /** * Main program * @param args + * @throws Exception */ - public static void main(String [] args) - throws IOException, MasterNotRunningException { + public static void main(String [] args) throws Exception { // create a fsck object Configuration conf = HBaseConfiguration.create(); conf.set("fs.defaultFS", conf.get("hbase.rootdir")); HBaseFsck fsck = new HBaseFsck(conf); - // Process command-line args. + // Process command-line args. for (int i = 0; i < args.length; i++) { String cmd = args[i]; if (cmd.equals("-details")) { Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (revision 1048879) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (working copy) @@ -26,99 +26,83 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; -import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.ipc.HMasterInterface; import org.apache.hadoop.hbase.ipc.HRegionInterface; import org.apache.hadoop.hbase.zookeeper.ZKAssign; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; public class HBaseFsckRepair { + /** + * Fix dupe assignment by doing silent closes on each RS hosting the region + * and then force ZK unassigned node to OFFLINE to trigger assignment by + * master. + * @param conf + * @param region + * @param servers + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ public static void fixDupeAssignment(Configuration conf, HRegionInfo region, List servers) - throws IOException { + throws IOException, KeeperException, InterruptedException { HRegionInfo actualRegion = new HRegionInfo(region); - // Clear status in master and zk - clearInMaster(conf, actualRegion); - clearInZK(conf, actualRegion); - - // Close region on the servers + // Close region on the servers silently for(HServerAddress server : servers) { - closeRegion(conf, server, actualRegion); + closeRegionSilentlyAndWait(conf, server, actualRegion); } - // It's unassigned so fix it as such - fixUnassigned(conf, actualRegion); + // Force ZK node to OFFLINE so master assigns + forceOfflineInZK(conf, actualRegion); } + /** + * Fix unassigned by creating/transition the unassigned ZK node for this + * region to OFFLINE state with a special flag to tell the master that this + * is a forced operation by HBCK. + * @param conf + * @param region + * @throws IOException + * @throws KeeperException + */ public static void fixUnassigned(Configuration conf, HRegionInfo region) - throws IOException { - + throws IOException, KeeperException { HRegionInfo actualRegion = new HRegionInfo(region); - // Clear status in master and zk - clearInMaster(conf, actualRegion); - clearInZK(conf, actualRegion); - - // Clear assignment in META or ROOT - clearAssignment(conf, actualRegion); + // Force ZK node to OFFLINE so master assigns + forceOfflineInZK(conf, actualRegion); } - private static void clearInMaster(Configuration conf, HRegionInfo region) - throws IOException { - System.out.println("Region being cleared in master: " + region); - HMasterInterface master = HConnectionManager.getConnection(conf).getMaster(); - long masterVersion = - master.getProtocolVersion("org.apache.hadoop.hbase.ipc.HMasterInterface", 25); - System.out.println("Master protocol version: " + masterVersion); - try { - // TODO: Do we want to do it this way? - // Better way is to tell master to fix the issue itself? - // That way it can use in-memory state to determine best plan -// master.clearFromTransition(region); - } catch (Exception e) {} + private static void forceOfflineInZK(Configuration conf, HRegionInfo region) + throws ZooKeeperConnectionException, KeeperException, IOException { + ZKAssign.createOrForceNodeOffline( + HConnectionManager.getConnection(conf).getZooKeeperWatcher(), + region, HConstants.HBCK_CODE_NAME); } - private static void clearInZK(Configuration conf, HRegionInfo region) - throws IOException { - ZooKeeperWatcher zkw = - HConnectionManager.getConnection(conf).getZooKeeperWatcher(); - try { - ZKAssign.deleteNodeFailSilent(zkw, region); - } catch (KeeperException e) { - throw new IOException("Unexpected ZK exception", e); - } - } - - private static void closeRegion(Configuration conf, HServerAddress server, - HRegionInfo region) - throws IOException { + private static void closeRegionSilentlyAndWait(Configuration conf, + HServerAddress server, HRegionInfo region) + throws IOException, InterruptedException { HRegionInterface rs = HConnectionManager.getConnection(conf).getHRegionConnection(server); rs.closeRegion(region, false); - } - - private static void clearAssignment(Configuration conf, - HRegionInfo region) - throws IOException { - HTable ht = null; - if (region.isMetaTable()) { - // Clear assignment in ROOT - ht = new HTable(conf, HConstants.ROOT_TABLE_NAME); + long timeout = conf.getLong("hbase.hbck.close.timeout", 120000); + long expiration = timeout + System.currentTimeMillis(); + while (System.currentTimeMillis() < expiration) { + try { + HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName()); + if (rsRegion == null) throw new NotServingRegionException(); + } catch (Exception e) { + return; + } + Thread.sleep(1000); } - else { - // Clear assignment in META - ht = new HTable(conf, HConstants.META_TABLE_NAME); - } - Delete del = new Delete(region.getRegionName()); - del.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER); - del.deleteColumns(HConstants.CATALOG_FAMILY, - HConstants.STARTCODE_QUALIFIER); - ht.delete(del); + throw new IOException("Region " + region + " failed to close within" + + " timeout " + timeout); } } \ No newline at end of file