commit b733da20769561d37d1d8465e55a06c11dedf512 Author: tedyu Date: Thu Oct 23 16:55:53 2014 -0700 HBASE-10955 HBCK leaves the region in masters in-memory RegionStates if region hdfs dir is lost (Enis) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index fc4b1d2..722c733 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -523,6 +523,7 @@ public class HBaseFsck extends Configured { public byte [] metaLastKey; public byte [] storesFirstKey; public byte [] storesLastKey; + @Override public String toString () { return "regionName=" + Bytes.toStringBinary(regionName) + "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) + @@ -1956,7 +1957,7 @@ public class HBaseFsck extends Configured { // these problems from META. if (shouldFixAssignments()) { errors.print("Trying to fix unassigned region..."); - closeRegion(hbi);// Close region will cause RS to abort. + undeployRegions(hbi); } if (shouldFixMeta()) { // wait for it to complete @@ -2111,13 +2112,13 @@ public class HBaseFsck extends Configured { // rename the contained into the container. FileSystem fs = targetRegionDir.getFileSystem(getConf()); FileStatus[] dirs = null; - try { + try { dirs = fs.listStatus(contained.getHdfsRegionDir()); } catch (FileNotFoundException fnfe) { // region we are attempting to merge in is not present! Since this is a merge, there is // no harm skipping this region if it does not exist. if (!fs.exists(contained.getHdfsRegionDir())) { - LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() + LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() + " is missing. Assuming already sidelined or moved."); } else { sidelineRegionDir(fs, contained); @@ -2127,7 +2128,7 @@ public class HBaseFsck extends Configured { if (dirs == null) { if (!fs.exists(contained.getHdfsRegionDir())) { - LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() + LOG.warn("[" + thread + "] HDFS region dir " + contained.getHdfsRegionDir() + " already sidelined."); } else { sidelineRegionDir(fs, contained); @@ -2177,20 +2178,20 @@ public class HBaseFsck extends Configured { static class WorkItemOverlapMerge implements Callable { private TableIntegrityErrorHandler handler; Collection overlapgroup; - + WorkItemOverlapMerge(Collection overlapgroup, TableIntegrityErrorHandler handler) { this.handler = handler; this.overlapgroup = overlapgroup; } - + @Override public Void call() throws Exception { handler.handleOverlapGroup(overlapgroup); return null; } }; - - + + /** * Maintain information about a particular table. */ @@ -2409,7 +2410,7 @@ public class HBaseFsck extends Configured { HTableDescriptor htd = getTableInfo().getHTD(); HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); - LOG.info("Plugged hold by creating new empty region: "+ newRegion + " " +region); + LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region); fixes++; } @@ -2421,7 +2422,7 @@ public class HBaseFsck extends Configured { * Cases: * - Clean regions that overlap * - Only .oldlogs regions (can't find start/stop range, or figure out) - * + * * This is basically threadsafe, except for the fixer increment in mergeOverlaps. */ @Override @@ -2704,11 +2705,11 @@ public class HBaseFsck extends Configured { private boolean handleOverlapsParallel(TableIntegrityErrorHandler handler, byte[] prevKey) throws IOException { // we parallelize overlap handler for the case we have lots of groups to fix. We can - // safely assume each group is independent. + // safely assume each group is independent. List merges = new ArrayList(overlapGroups.size()); List> rets; for (Collection overlap : overlapGroups.asMap().values()) { - // + // merges.add(new WorkItemOverlapMerge(overlap, handler)); } try { @@ -4022,7 +4023,7 @@ public class HBaseFsck extends Configured { return hbck.getRetCode(); } }; - + public HBaseFsck exec(ExecutorService exec, String[] args) throws KeeperException, IOException, ServiceException, InterruptedException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java index ed3a962..fefdc20 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java @@ -43,7 +43,6 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.zookeeper.KeeperException; /** @@ -203,12 +202,10 @@ public class HBaseFsckRepair { HRegionInfo hri, HTableDescriptor htd) throws IOException { // Create HRegion Path root = FSUtils.getRootDir(conf); - HRegion region = HRegion.createHRegion(hri, root, conf, htd); - HLog hlog = region.getLog(); + HRegion region = HRegion.createHRegion(hri, root, conf, htd, null); // Close the new region to flush to disk. Close log file too. region.close(); - hlog.closeAndDelete(); return region; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index d769f4e..4fb0a7b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -1681,6 +1681,34 @@ public class TestHBaseFsck { } /** + * This creates and fixes a bad table with a missing region which is the 1st region -- hole in + * meta and data missing in the fs. + */ + @Test(timeout=120000) + public void testRegionDeployedNotInHdfs() throws Exception { + TableName table = + TableName.valueOf("testSingleRegionDeployedNotInHdfs"); + try { + setupTable(table); + TEST_UTIL.getHBaseAdmin().flush(table.getName()); + + // Mess it up by deleting region dir + deleteRegion(conf, tbl.getTableDescriptor(), + HConstants.EMPTY_START_ROW, Bytes.toBytes("A"), false, + false, true); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS }); + // fix hole + doFsck(conf, true); + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + } finally { + deleteTable(table); + } + } + + /** * This creates and fixes a bad table with missing last region -- hole in meta and data missing in * the fs. */