Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1149081) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy) @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -311,7 +312,7 @@ MetaEntry m = new MetaEntry(rootLocation.getRegionInfo(), rootLocation.getServerAddress(), null, System.currentTimeMillis()); - HbckInfo hbInfo = new HbckInfo(m); + HbckInfo hbInfo = new HbckInfo(m, INFO_FROM.FROM_META); regionInfo.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo); return true; } @@ -469,8 +470,13 @@ * Checks tables integrity. Goes over all regions and scans the tables. * Collects all the pieces for each table and checks if there are missing, * repeated or overlapping ones. + * + * @throws KeeperException + * @throws IOException + * @throws InterruptedException */ - void checkIntegrity() { + void checkIntegrity() throws IOException, KeeperException, + InterruptedException { for (HbckInfo hbi : regionInfo.values()) { // Check only valid, working regions if (hbi.metaEntry == null) continue; @@ -482,7 +488,12 @@ // the region chain in META //if (hbi.foundRegionDir == null) continue; //if (hbi.deployedOn.size() != 1) continue; - if (hbi.deployedOn.size() == 0) continue; + + // no deployed is checked elsewhere. Include + // these cases in modTInfo, so we can evaluate those regions as part of + // the region chain in META.(delete this regioninfo from META when in checkConsistency(),it will make hole in chain + // and we can read regions from HDFS or make a new region to fix this hole.) + //if (hbi.deployedOn.size() == 0) continue; // We should be safe here String tableName = hbi.metaEntry.getTableDesc().getNameAsString(); @@ -541,20 +552,39 @@ * Check the region chain (from META) of this table. We are looking for * holes, overlaps, and cycles. * @return false if there are errors + * @throws KeeperException + * @throws IOException + * @throws InterruptedException */ - public boolean checkRegionChain() { + public boolean checkRegionChain() throws IOException, KeeperException, + InterruptedException { Collections.sort(regions); HbckInfo last = null; int originalErrorsCount = errors.getErrorList().size(); + boolean haveLast = false; // true if the the last Region of the table is + // end of HConstants.EMPTY_BYTE_ARRAY for (HbckInfo r : regions) { if (last == null) { // This is the first region, check that the start key is empty - if (! Bytes.equals(r.metaEntry.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) { + // if it is not empty, it is a hole problem,call HoleFixer to fix. + if (!Bytes.equals(r.metaEntry.getStartKey(), + HConstants.EMPTY_BYTE_ARRAY)) { errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, - "First region should start with an empty key.", - this, r); + "First region should start with an empty key.", this, r); + + if (shouldFix()) { + errors.print("Trying to fix a problem with .META..."); + setShouldRerun(); + HBaseFsckRepair.HoleFixer(conf, null, r, regionInfo); + } } + + // just one region in this table,check the end Key. + if (Bytes + .equals(r.metaEntry.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)) { + haveLast = true; + } } else { // Check if endKey < startKey @@ -573,6 +603,9 @@ Bytes.toStringBinary(r.metaEntry.getEndKey())), this, r, last); } + } else { + // one region is end of empty + haveLast = true; } // Check if the startkeys are different @@ -588,8 +621,13 @@ if (cmp > 0) { // hole errors.reportError(ERROR_CODE.HOLE_IN_REGION_CHAIN, - "There is a hole in the region chain.", - this, r, last); + "There is a hole in the region chain.", this, r, last); + // call HoleFixer to fix hole problem + if (shouldFix()) { + errors.print("Trying to fix a problem with .META..."); + setShouldRerun(); + HBaseFsckRepair.HoleFixer(conf, last, r, regionInfo); + } } else if (cmp < 0) { // overlap errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN, @@ -603,6 +641,18 @@ last = r; } + // no region end of empty, it is a hole problem,call HoleFixer to fix. + if (!haveLast) { + errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, + "Last region should end with an empty key.", this, last); + + if (shouldFix()) { + errors.print("Trying to fix a problem with .META..."); + setShouldRerun(); + HBaseFsckRepair.HoleFixer(conf, last, null, regionInfo); + } + } + return errors.getErrorList().size() == originalErrorsCount; } @@ -642,10 +692,10 @@ * region name. If the region has not been seen yet, a new entry is added * and returned. */ - private synchronized HbckInfo getOrCreateInfo(String name) { + private synchronized HbckInfo getOrCreateInfo(String name, INFO_FROM infoFrom) { HbckInfo hbi = regionInfo.get(name); if (hbi == null) { - hbi = new HbckInfo(null); + hbi = new HbckInfo(null, infoFrom); regionInfo.put(name, hbi); } return hbi; @@ -754,7 +804,7 @@ startCode = value; } MetaEntry m = new MetaEntry(info, server, startCode, ts); - HbckInfo hbInfo = new HbckInfo(m); + HbckInfo hbInfo = new HbckInfo(m, INFO_FROM.FROM_META); HbckInfo previous = regionInfo.put(info.getEncodedName(), hbInfo); if (previous != null) { throw new IOException("Two entries in META are same " + previous); @@ -785,7 +835,7 @@ /** * Stores the entries scanned from META */ - private static class MetaEntry extends HRegionInfo { + public static class MetaEntry extends HRegionInfo { HServerAddress regionServer; // server hosting this region long modTime; // timestamp of most recent modification metadata @@ -806,8 +856,14 @@ FileStatus foundRegionDir = null; List deployedOn = Lists.newArrayList(); - HbckInfo(MetaEntry metaEntry) { + INFO_FROM infoFrom = INFO_FROM.FROM_META; // where the RegionInfo from. META + // table? RegionServer? HDFS? + + HRegionInfo regionInfoFromRSOrHDFS = null;// if the RegionInfo from RS or + // HDFS, put the info in it. + HbckInfo(MetaEntry metaEntry, INFO_FROM infoFrom) { this.metaEntry = metaEntry; + this.infoFrom = infoFrom; } public synchronized void addServer(HServerAddress server) { @@ -856,12 +912,13 @@ } interface ErrorReporter { + // add two error code:LAST_REGION_ENDKEY_NOT_EMPTY and HDFS_GET_FAILURE public static enum ERROR_CODE { UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED, MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE, FIRST_REGION_STARTKEY_NOT_EMPTY, DUPE_STARTKEYS, - HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE + HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, LAST_REGION_ENDKEY_NOT_EMPTY, HDFS_GET_FAILURE } public void clear(); public void report(String message); @@ -1029,7 +1086,11 @@ // check to see if the existence of this region matches the region in META for (HRegionInfo r:regions) { - HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName()); + HbckInfo hbi = hbck.getOrCreateInfo(r.getEncodedName(), + INFO_FROM.FROM_RS); + if (hbi.infoFrom == INFO_FROM.FROM_RS) { + hbi.regionInfoFromRSOrHDFS = r; + } hbi.addServer(rsinfo.getServerAddress()); } } catch (IOException e) { // unable to connect to the region server. @@ -1065,6 +1126,41 @@ return done; } + /** + * get the regioninfo from hdfs. + * + * @param regionDir region data path + * @return regioninfo + */ + HRegionInfo getRegionInfoFromHDFS(FileStatus regionDir) { + FSDataInputStream is = null; + HRegionInfo hri = null; + Path regioninfofile = new Path(regionDir.getPath(), ".regioninfo"); + try { + if (!fs.exists(regioninfofile)) { + errors.reportError(ERROR_CODE.HDFS_GET_FAILURE, + regioninfofile.toString() + " is not exist. "); + return null; + } + + is = fs.open(regioninfofile); + hri = new HRegionInfo(); + hri.readFields(is); + } catch (IOException e) { + errors.reportError(ERROR_CODE.HDFS_GET_FAILURE, + "read the regioninfo from " + regioninfofile.toString() + + " is fail." + e); + return null; + } finally { + try { + is.close(); + } catch (IOException e) { + errors.reportError(ERROR_CODE.HDFS_GET_FAILURE, "close " + + regioninfofile.toString() + " is fail." + e); + } + } + return hri; + } @Override public synchronized void run() { try { @@ -1081,7 +1177,7 @@ // ignore directories that aren't hexadecimal if (!encodedName.toLowerCase().matches("[0-9a-f]+")) continue; - HbckInfo hbi = hbck.getOrCreateInfo(encodedName); + HbckInfo hbi = hbck.getOrCreateInfo(encodedName, INFO_FROM.FROM_HDFS); synchronized (hbi) { if (hbi.foundRegionDir != null) { errors.print("Directory " + encodedName + " duplicate??" + @@ -1089,6 +1185,10 @@ } hbi.foundRegionDir = regionDir; + if (hbi.infoFrom == INFO_FROM.FROM_HDFS) { + HRegionInfo hri = getRegionInfoFromHDFS(regionDir); + hbi.regionInfoFromRSOrHDFS = hri; + } // Set a flag if this region contains only edits // This is special case if a region is left after split hbi.onlyEdits = true; @@ -1254,5 +1354,15 @@ Runtime.getRuntime().exit(code); } + + /** + * where the regioninfo in regionInfo list from. + */ + private enum INFO_FROM { + FROM_META, // from meta table + FROM_RS, // from regionserver + FROM_HDFS + // from hdfs + } } Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (revision 1149081) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (working copy) @@ -20,16 +20,25 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.ipc.HMasterInterface; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.zookeeper.KeeperException; @@ -62,6 +71,85 @@ } /** + * to fix hole problem. + * + * @param conf + * @param leftRegionInfo the left edge of the hole + * @param rightRegionInfo the right edge of the hole + * @param regionInfo regionInfo list,all info in it. + * @throws IOException + * @throws KeeperException + * @throws InterruptedException + */ + public static void HoleFixer(Configuration conf, HbckInfo leftRegionInfo, + HbckInfo rightRegionInfo, TreeMap regionInfo) + throws IOException, KeeperException, InterruptedException { + // check param + if (conf == null || regionInfo == null) { + return; + } + + List weaveHoleList = null; + HTable metatable = new HTable(conf, HConstants.META_TABLE_NAME); + HMasterInterface HMasterInFe = HConnectionManager.getConnection(conf) + .getMaster(); + + // get the regioninfo from regionserver or hdfs that the key range is + // overlap with the hole range. + weaveHoleList = getWeaveHoleList(conf, leftRegionInfo, rightRegionInfo, + regionInfo); + + // some regioninfo are in hole range,so assign these regioninfo. + if (weaveHoleList != null && weaveHoleList.size() > 0) { + for (HbckInfo hki : weaveHoleList) { + Put p = new Put(hki.regionInfoFromRSOrHDFS.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hki.regionInfoFromRSOrHDFS)); + metatable.put(p); + + // if it is deployed on RS, unassign it first. + if (hki.deployedOn.size() > 0) { + fixDupeAssignment(conf, hki.regionInfoFromRSOrHDFS, hki.deployedOn); + } + + // if it is not deployed on RS, but in hdfs,assign it. + else if (hki.foundRegionDir != null) { + fixUnassigned(conf, hki.regionInfoFromRSOrHDFS); + } + } + } else { + // if there are no region that (in RS or HDFS) the key range is overlap + // with the hole range. + // create a new region to fix the hole. + byte[] startKey = null; + byte[] endKey = null; + HbckInfo tempRegioninfo = leftRegionInfo; + + // get the new region's startKey and endKey. + if (leftRegionInfo == null) { + startKey = HConstants.EMPTY_BYTE_ARRAY; + endKey = rightRegionInfo.metaEntry.getStartKey(); + tempRegioninfo = rightRegionInfo; + } else if (rightRegionInfo == null) { + startKey = leftRegionInfo.metaEntry.getEndKey(); + endKey = HConstants.EMPTY_BYTE_ARRAY; + } else { + startKey = leftRegionInfo.metaEntry.getEndKey(); + endKey = rightRegionInfo.metaEntry.getStartKey(); + } + + HRegionInfo hri = new HRegionInfo( + tempRegioninfo.metaEntry.getTableDesc(), startKey, endKey); + Put p = new Put(hri.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + // put it into META and assign it. + metatable.put(p); + HMasterInFe.assign(hri.getRegionName(), false); + } + } + + /** * Fix unassigned by creating/transition the unassigned ZK node for this * region to OFFLINE state with a special flag to tell the master that this * is a forced operation by HBCK. @@ -105,4 +193,118 @@ throw new IOException("Region " + region + " failed to close within" + " timeout " + timeout); } + /** + * search the region(in RS or in HDFS) that the key range is overlap with the + * hole range. + * + * if the hole range is [10,20],then the region (in RS or in HDFS)that key + * range is [0, 11] or [15,21]or [15,18]or [5,25] will be put ito the + * weaveHoleList to fix the hole. + * + * + * @param conf + * @param leftRegionInfo + * @param rightRegionInfo + * @param regionInfo + * @return + */ + private static List getWeaveHoleList(Configuration conf, + HbckInfo leftRegionInfo, HbckInfo rightRegionInfo, + TreeMap regionInfo) { + HbckInfo tempRegioninfo = leftRegionInfo; + + // check param + if (conf == null || regionInfo == null) { + return null; + } + + // the leftRegionInfo and rightRegionInfo are not both null. + if (leftRegionInfo == null) { + tempRegioninfo = rightRegionInfo; + } + + List weaveHoleList = new ArrayList(); + + Iterator it = regionInfo.keySet().iterator(); + String key = null; + HbckInfo value = null; + HTableDescriptor talbeDescRight = tempRegioninfo.metaEntry.getTableDesc(); + byte[] leftEndKey = leftRegionInfo == null ? HConstants.EMPTY_BYTE_ARRAY + : leftRegionInfo.metaEntry.getEndKey(); + byte[] rightStartKey = rightRegionInfo == null ? HConstants.EMPTY_BYTE_ARRAY + : rightRegionInfo.metaEntry.getStartKey(); + + while (it.hasNext()) { + key = (String) it.next(); + value = regionInfo.get(key); + if (value == null + || value.regionInfoFromRSOrHDFS == null + || talbeDescRight.compareTo(value.regionInfoFromRSOrHDFS + .getTableDesc()) != 0) { + continue; + } + + // true if the region(in RS or in HDFS) that the key range is overlap with + // the hole range + if (isOverChain(value.regionInfoFromRSOrHDFS, leftEndKey, rightStartKey)) { + weaveHoleList.add(value); + } + } + + return weaveHoleList; + } + + /** + * + * @param insetRegionInfo + * @param leftKey + * @param rightKey + * @return + */ + // is overlap the hole range? + private static boolean isOverChain(HRegionInfo insetRegionInfo, + byte[] leftKey, byte[] rightKey) { + // First region is not start with an empty key + if (Arrays.equals(leftKey, HConstants.EMPTY_BYTE_ARRAY)) { + if (Bytes.compareTo(insetRegionInfo.getStartKey(), rightKey) <= 0) { + return true; + } + } + + // last region is not end with an empty key + if (Arrays.equals(rightKey, HConstants.EMPTY_BYTE_ARRAY)) { + if (Bytes.compareTo(insetRegionInfo.getEndKey(), leftKey) >= 0 + || Bytes.compareTo(insetRegionInfo.getEndKey(), + HConstants.EMPTY_BYTE_ARRAY) == 0) { + return true; + } + } + + // i.e: hole range [5,10], return true,if the region range is [3,6] + if (Bytes.compareTo(insetRegionInfo.getStartKey(), leftKey) <= 0 + && Bytes.compareTo(insetRegionInfo.getEndKey(), leftKey) >= 0) { + return true; + } + + // i.e: hole range [5,10], return true,if the region range is [6,12] + if (Bytes.compareTo(insetRegionInfo.getStartKey(), rightKey) <= 0 + && Bytes.compareTo(insetRegionInfo.getEndKey(), rightKey) >= 0) { + return true; + } + + // i.e: hole range [5,10], return true,if the region range is [6,9] + if (Bytes.compareTo(insetRegionInfo.getStartKey(), leftKey) >= 0 + && Bytes.compareTo(insetRegionInfo.getEndKey(), rightKey) <= 0) { + return true; + } + + // i.e: hole range [5,10], return true,if the region range is [4,11] + if (Bytes.compareTo(insetRegionInfo.getStartKey(), leftKey) <= 0 + && Bytes.compareTo(insetRegionInfo.getEndKey(), rightKey) >= 0) { + return true; + } + + return false; + } + } \ No newline at end of file