Index: src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
===================================================================
--- src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (revision 1044483)
+++ src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (working copy)
@@ -19,7 +19,7 @@
*/
package org.apache.hadoop.hbase.util;
-import java.io.IOException;
+import static org.junit.Assert.assertEquals;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,8 +36,6 @@
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
public class TestHBaseFsck {
final Log LOG = LogFactory.getLog(getClass());
@@ -53,7 +51,7 @@
}
@Test
- public void testHBaseFsck() throws IOException {
+ public void testHBaseFsck() throws Exception {
HBaseFsck fsck = new HBaseFsck(conf);
fsck.displayFullReport();
fsck.setTimeLag(0);
@@ -71,7 +69,7 @@
// point to a different region server
HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
ResultScanner scanner = meta.getScanner(new Scan());
-
+
resforloop : for (Result res : scanner) {
long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER));
@@ -100,6 +98,7 @@
// Fixed or not, it still reports inconsistencies
assertEquals(-1, result);
+ Thread.sleep(15000);
// Disabled, won't work because the region stays unassigned, see HBASE-3217
// new HTable(conf, TABLE).getScanner(new Scan());
}
Index: src/main/java/org/apache/hadoop/hbase/HConstants.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/HConstants.java (revision 1044483)
+++ src/main/java/org/apache/hadoop/hbase/HConstants.java (working copy)
@@ -358,6 +358,9 @@
public static final String
REPLICATION_ENABLE_KEY = "hbase.replication";
+ /** HBCK special code name used as server name when manipulating ZK nodes */
+ public static final String HBCK_CODE_NAME = "HBCKsecret123";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1044483)
+++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy)
@@ -40,6 +40,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
@@ -51,10 +52,9 @@
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.catalog.RootLocationEditor;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.executor.EventHandler.EventType;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.RegionTransitionData;
-import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
+import org.apache.hadoop.hbase.executor.EventHandler.EventType;
import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan;
import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler;
import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
@@ -65,9 +65,9 @@
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
import org.apache.hadoop.hbase.zookeeper.ZKTable;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.NodeAndData;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.zookeeper.AsyncCallback;
@@ -143,7 +143,7 @@
* @param serverManager
* @param catalogTracker
* @param service
- * @throws KeeperException
+ * @throws KeeperException
*/
public AssignmentManager(Server master, ServerManager serverManager,
CatalogTracker catalogTracker, final ExecutorService service)
@@ -337,6 +337,11 @@
LOG.warn("Unexpected NULL input " + data);
return;
}
+ // Check if this is a special HBCK transition
+ if (data.getServerName().equals(HConstants.HBCK_CODE_NAME)) {
+ handleHBCK(data);
+ return;
+ }
// Verify this is a known server
if (!serverManager.isServerOnline(data.getServerName()) &&
!this.master.getServerName().equals(data.getServerName())) {
@@ -424,6 +429,45 @@
}
}
+ /**
+ * Handle a ZK unassigned node transition triggered by HBCK repair tool.
+ *
+ * This is handled in a separate code path because it breaks the normal rules.
+ * @param data
+ */
+ private void handleHBCK(RegionTransitionData data) {
+ String encodedName = HRegionInfo.encodeRegionName(data.getRegionName());
+ LOG.info("Handling HBCK triggered transition=" + data.getEventType() +
+ ", server=" + data.getServerName() + ", region=" +
+ HRegionInfo.prettyPrint(encodedName));
+ RegionState regionState = regionsInTransition.get(encodedName);
+ switch (data.getEventType()) {
+ case M_ZK_REGION_OFFLINE:
+ HRegionInfo regionInfo = null;
+ if (regionState != null) {
+ regionInfo = regionState.getRegion();
+ } else {
+ try {
+ regionInfo = MetaReader.getRegion(catalogTracker,
+ data.getRegionName()).getFirst();
+ } catch (IOException e) {
+ LOG.info("Exception reading META doing HBCK repair operation", e);
+ return;
+ }
+ }
+ LOG.info("HBCK repair is triggering assignment of region=" +
+ regionInfo.getRegionNameAsString());
+ // trigger assign, node is already in OFFLINE so don't need to update ZK
+ assign(regionInfo, false);
+ break;
+
+ default:
+ LOG.warn("Received unexpected region state from HBCK (" +
+ data.getEventType() + ")");
+ break;
+ }
+ }
+
// ZooKeeper events
/**
@@ -1001,7 +1045,7 @@
public void unassign(HRegionInfo region, boolean force) {
LOG.debug("Starting unassignment of region " +
region.getRegionNameAsString() + " (offlining)");
- synchronized (this.regions) {
+ synchronized (this.regions) {
// Check if this region is currently assigned
if (!regions.containsKey(region)) {
LOG.debug("Attempted to unassign region " +
@@ -1610,9 +1654,9 @@
/**
* Process shutdown server removing any assignments.
* @param hsi Server that went down.
- * @return set of regions on this server that are not in transition
+ * @return list of regions in transition on this server
*/
- public List processServerShutdown(final HServerInfo hsi) {
+ public List processServerShutdown(final HServerInfo hsi) {
// Clean out any existing assignment plans for this server
synchronized (this.regionPlans) {
for (Iterator > i =
@@ -1628,7 +1672,7 @@
// Remove this server from map of servers to regions, and remove all regions
// of this server from online map of regions.
Set deadRegions = null;
- List rits = new ArrayList();
+ List rits = new ArrayList();
synchronized (this.regions) {
List assignedRegions = this.servers.remove(hsi);
if (assignedRegions == null || assignedRegions.isEmpty()) {
@@ -1646,7 +1690,7 @@
synchronized (regionsInTransition) {
for (RegionState region : this.regionsInTransition.values()) {
if (deadRegions.remove(region.getRegion())) {
- rits.add(region.getRegion());
+ rits.add(region);
}
}
}
Index: src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (revision 1044483)
+++ src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (working copy)
@@ -40,6 +40,7 @@
import org.apache.hadoop.hbase.master.DeadServer;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
import org.apache.zookeeper.KeeperException;
@@ -98,7 +99,7 @@
// doing after log splitting. Could do some states before -- OPENING?
// OFFLINE? -- and then others after like CLOSING that depend on log
// splitting.
- List regionsInTransition =
+ List regionsInTransition =
this.services.getAssignmentManager().processServerShutdown(this.hsi);
// Assign root and meta if we were carrying them.
@@ -133,11 +134,16 @@
}
}
- // Remove regions that were in transition
- for (HRegionInfo rit : regionsInTransition) hris.remove(rit);
- LOG.info("Reassigning the " + hris.size() + " region(s) that " + serverName
+ // Skip regions that were in transition unless CLOSING or PENDING_CLOSE
+ for (RegionState rit : regionsInTransition) {
+ if (!rit.isClosing() && !rit.isPendingClose()) {
+ hris.remove(rit.getRegion());
+ }
+ }
+
+ LOG.info("Reassigning " + hris.size() + " region(s) that " + serverName
+ " was carrying (skipping " + regionsInTransition.size() +
- " regions(s) that are in transition)");
+ " regions(s) that are already in transition)");
// Iterate regions that were on this server and assign them
for (Map.Entry e: hris.entrySet()) {
Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1044483)
+++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy)
@@ -23,11 +23,11 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.HashSet;
-import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -50,18 +50,17 @@
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.MetaScanner;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
-import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
+import org.apache.zookeeper.KeeperException;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
/**
- * Check consistency among the in-memory states of the master and the
+ * Check consistency among the in-memory states of the master and the
* region server(s) and the state of data in HDFS.
*/
public class HBaseFsck {
@@ -92,7 +91,7 @@
* @throws MasterNotRunningException if the master is not running
* @throws ZooKeeperConnectionException if unable to connect to zookeeper
*/
- public HBaseFsck(Configuration conf)
+ public HBaseFsck(Configuration conf)
throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
this.conf = conf;
@@ -105,8 +104,10 @@
* Contacts the master and prints out cluster-wide information
* @throws IOException if a remote or network exception occurs
* @return 0 on success, non-zero on failure
+ * @throws KeeperException
+ * @throws InterruptedException
*/
- int doWork() throws IOException {
+ int doWork() throws IOException, KeeperException, InterruptedException {
// print hbase server version
errors.print("Version: " + status.getHBaseVersion());
@@ -152,7 +153,7 @@
// From the master, get a list of all known live region servers
Collection regionServers = status.getServerInfo();
- errors.print("Number of live region servers: " +
+ errors.print("Number of live region servers: " +
regionServers.size());
if (details) {
for (HServerInfo rsinfo: regionServers) {
@@ -162,7 +163,7 @@
// From the master, get a list of all dead region servers
Collection deadRegionServers = status.getDeadServerNames();
- errors.print("Number of dead region servers: " +
+ errors.print("Number of dead region servers: " +
deadRegionServers.size());
if (details) {
for (String name: deadRegionServers) {
@@ -280,7 +281,7 @@
regionInfo.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo);
return true;
}
-
+
/**
* Contacts each regionserver and fetches metadata about regions.
* @param regionServerList - the list of region servers to connect to
@@ -315,7 +316,7 @@
HbckInfo hbi = getOrCreateInfo(r.getEncodedName());
hbi.deployedOn.add(rsinfo.getServerAddress());
}
- } catch (IOException e) { // unable to connect to the region server.
+ } catch (IOException e) { // unable to connect to the region server.
errors.reportError("\nRegionServer:" + rsinfo.getServerName() +
" Unable to fetch region information. " + e);
}
@@ -324,8 +325,11 @@
/**
* Check consistency of all regions that have been found in previous phases.
+ * @throws KeeperException
+ * @throws InterruptedException
*/
- void checkConsistency() throws IOException {
+ void checkConsistency()
+ throws IOException, KeeperException, InterruptedException {
for (java.util.Map.Entry e: regionInfo.entrySet()) {
doConsistencyCheck(e.getKey(), e.getValue());
}
@@ -333,9 +337,11 @@
/**
* Check a single region for consistency and correct deployment.
+ * @throws KeeperException
+ * @throws InterruptedException
*/
void doConsistencyCheck(final String key, final HbckInfo hbi)
- throws IOException {
+ throws IOException, KeeperException, InterruptedException {
String descriptiveName = hbi.toString();
boolean inMeta = hbi.metaEntry != null;
@@ -533,8 +539,8 @@
/**
* Return a list of user-space table names whose metadata have not been
* modified in the last few milliseconds specified by timelag
- * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
- * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
+ * if any of the REGIONINFO_QUALIFIER, SERVER_QUALIFIER, STARTCODE_QUALIFIER,
+ * SPLITA_QUALIFIER, SPLITB_QUALIFIER have not changed in the last
* milliseconds specified by timelag, then the table is a candidate to be returned.
* @param regionList - all entries found in .META
* @return tables that have not been modified recently
@@ -580,8 +586,11 @@
* If there are inconsistencies (i.e. zero or more than one regions
* pretend to be holding the .META.) try to fix that and report an error.
* @throws IOException from HBaseFsckRepair functions
+ * @throws KeeperException
+ * @throws InterruptedException
*/
- boolean checkMetaEntries() throws IOException {
+ boolean checkMetaEntries()
+ throws IOException, KeeperException, InterruptedException {
List metaRegions = Lists.newArrayList();
for (HbckInfo value : regionInfo.values()) {
if (value.metaEntry.isMetaTable()) {
@@ -709,7 +718,7 @@
HServerAddress regionServer; // server hosting this region
long modTime; // timestamp of most recent modification metadata
- public MetaEntry(HRegionInfo rinfo, HServerAddress regionServer,
+ public MetaEntry(HRegionInfo rinfo, HServerAddress regionServer,
byte[] startCode, long modTime) {
super(rinfo);
this.regionServer = regionServer;
@@ -883,16 +892,16 @@
/**
* Main program
* @param args
+ * @throws Exception
*/
- public static void main(String [] args)
- throws IOException, MasterNotRunningException {
+ public static void main(String [] args) throws Exception {
// create a fsck object
Configuration conf = HBaseConfiguration.create();
conf.set("fs.defaultFS", conf.get("hbase.rootdir"));
HBaseFsck fsck = new HBaseFsck(conf);
- // Process command-line args.
+ // Process command-line args.
for (int i = 0; i < args.length; i++) {
String cmd = args[i];
if (cmd.equals("-details")) {
Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (revision 1044483)
+++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (working copy)
@@ -26,99 +26,83 @@
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.zookeeper.ZKAssign;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
public class HBaseFsckRepair {
+ /**
+ * Fix dupe assignment by doing silent closes on each RS hosting the region
+ * and then force ZK unassigned node to OFFLINE to trigger assignment by
+ * master.
+ * @param conf
+ * @param region
+ * @param servers
+ * @throws IOException
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
public static void fixDupeAssignment(Configuration conf, HRegionInfo region,
List servers)
- throws IOException {
+ throws IOException, KeeperException, InterruptedException {
HRegionInfo actualRegion = new HRegionInfo(region);
- // Clear status in master and zk
- clearInMaster(conf, actualRegion);
- clearInZK(conf, actualRegion);
-
- // Close region on the servers
+ // Close region on the servers silently
for(HServerAddress server : servers) {
- closeRegion(conf, server, actualRegion);
+ closeRegionSilentlyAndWait(conf, server, actualRegion);
}
- // It's unassigned so fix it as such
- fixUnassigned(conf, actualRegion);
+ // Force ZK node to OFFLINE so master assigns
+ forceOfflineInZK(conf, actualRegion);
}
+ /**
+ * Fix unassigned by creating/transition the unassigned ZK node for this
+ * region to OFFLINE state with a special flag to tell the master that this
+ * is a forced operation by HBCK.
+ * @param conf
+ * @param region
+ * @throws IOException
+ * @throws KeeperException
+ */
public static void fixUnassigned(Configuration conf, HRegionInfo region)
- throws IOException {
-
+ throws IOException, KeeperException {
HRegionInfo actualRegion = new HRegionInfo(region);
- // Clear status in master and zk
- clearInMaster(conf, actualRegion);
- clearInZK(conf, actualRegion);
-
- // Clear assignment in META or ROOT
- clearAssignment(conf, actualRegion);
+ // Force ZK node to OFFLINE so master assigns
+ forceOfflineInZK(conf, actualRegion);
}
- private static void clearInMaster(Configuration conf, HRegionInfo region)
- throws IOException {
- System.out.println("Region being cleared in master: " + region);
- HMasterInterface master = HConnectionManager.getConnection(conf).getMaster();
- long masterVersion =
- master.getProtocolVersion("org.apache.hadoop.hbase.ipc.HMasterInterface", 25);
- System.out.println("Master protocol version: " + masterVersion);
- try {
- // TODO: Do we want to do it this way?
- // Better way is to tell master to fix the issue itself?
- // That way it can use in-memory state to determine best plan
-// master.clearFromTransition(region);
- } catch (Exception e) {}
+ private static void forceOfflineInZK(Configuration conf, HRegionInfo region)
+ throws ZooKeeperConnectionException, KeeperException, IOException {
+ ZKAssign.createOrForceNodeOffline(
+ HConnectionManager.getConnection(conf).getZooKeeperWatcher(),
+ region, HConstants.HBCK_CODE_NAME);
}
- private static void clearInZK(Configuration conf, HRegionInfo region)
- throws IOException {
- ZooKeeperWatcher zkw =
- HConnectionManager.getConnection(conf).getZooKeeperWatcher();
- try {
- ZKAssign.deleteNodeFailSilent(zkw, region);
- } catch (KeeperException e) {
- throw new IOException("Unexpected ZK exception", e);
- }
- }
-
- private static void closeRegion(Configuration conf, HServerAddress server,
- HRegionInfo region)
- throws IOException {
+ private static void closeRegionSilentlyAndWait(Configuration conf,
+ HServerAddress server, HRegionInfo region)
+ throws IOException, InterruptedException {
HRegionInterface rs =
HConnectionManager.getConnection(conf).getHRegionConnection(server);
rs.closeRegion(region, false);
- }
-
- private static void clearAssignment(Configuration conf,
- HRegionInfo region)
- throws IOException {
- HTable ht = null;
- if (region.isMetaTable()) {
- // Clear assignment in ROOT
- ht = new HTable(conf, HConstants.ROOT_TABLE_NAME);
+ long timeout = conf.getLong("hbase.hbck.close.timeout", 120000);
+ long expiration = timeout + System.currentTimeMillis();
+ while (System.currentTimeMillis() < expiration) {
+ try {
+ HRegionInfo rsRegion = rs.getRegionInfo(region.getRegionName());
+ if (rsRegion == null) throw new NotServingRegionException();
+ } catch (Exception e) {
+ return;
+ }
+ Thread.sleep(1000);
}
- else {
- // Clear assignment in META
- ht = new HTable(conf, HConstants.META_TABLE_NAME);
- }
- Delete del = new Delete(region.getRegionName());
- del.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
- del.deleteColumns(HConstants.CATALOG_FAMILY,
- HConstants.STARTCODE_QUALIFIER);
- ht.delete(del);
+ throw new IOException("Region " + region + " failed to close within" +
+ " timeout " + timeout);
}
}
\ No newline at end of file