Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1302529) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy) @@ -552,6 +552,13 @@ if (!foundVersionFile) { errors.reportError(ERROR_CODE.NO_VERSION_FILE, "Version file does not exist in root dir " + rootDir); + if (shouldFix()) { + LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME + + " file."); + setShouldRerun(); + FSUtils.setVersion(fs, rootDir, + conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); + } } // level 1: /* @@ -632,8 +639,9 @@ */ void checkConsistency() throws IOException, KeeperException, InterruptedException { + List metaLst = HBaseFsckRepair.getkMetaInfo(conf); for (java.util.Map.Entry e: regionInfo.entrySet()) { - doConsistencyCheck(e.getKey(), e.getValue()); + doConsistencyCheck(e.getKey(), e.getValue(), metaLst); } } @@ -642,8 +650,9 @@ * @throws KeeperException * @throws InterruptedException */ - void doConsistencyCheck(final String key, final HbckInfo hbi) - throws IOException, KeeperException, InterruptedException { + void doConsistencyCheck(final String key, final HbckInfo hbi, + List metaLst) throws IOException, KeeperException, + InterruptedException { String descriptiveName = hbi.toString(); boolean inMeta = hbi.metaEntry != null; @@ -686,10 +695,31 @@ errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region " + descriptiveName + " on HDFS, but not listed in META " + "or deployed on any region server"); + // load region info from hdfs. + if (null == hbi.metaEntry) { + loadMetaEntry(hbi); + } + if (shouldFix() + && HBaseFsckRepair.checkMetaTableHole(metaLst, hbi.metaEntry)) { + setShouldRerun(); + errors.print("Trying to plug the hole in .META, encodename is " + + hbi.metaEntry.getEncodedName()); + HBaseFsckRepair.fixupMetaTableHole(conf, hbi.metaEntry); + } } else if (!inMeta && inHdfs && isDeployed) { errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn)); - + // load region info from hdfs. + if (null == hbi.metaEntry) { + loadMetaEntry(hbi); + } + if (shouldFix() + && (HBaseFsckRepair.checkMetaTableHole(metaLst, hbi.metaEntry))) { + setShouldRerun(); + errors.print("Trying to plug the hole in .META, encodename is " + + hbi.metaEntry.getEncodedName()); + HBaseFsckRepair.fixupMetaTableHole(conf, hbi.metaEntry); + } // ========== Cases where the region is in META ============= } else if (inMeta && !inHdfs && !isDeployed) { errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region " @@ -712,6 +742,11 @@ errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, "Region " + descriptiveName + " should not be deployed according " + "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn)); + if (shouldFix()) { + errors.print("Trying to unassign the region " + descriptiveName); + setShouldRerun(); + admin.getMaster().unassign(hbi.metaEntry.getRegionName(), false); + } } else if (inMeta && inHdfs && isMultiplyDeployed) { errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName + " is listed in META on region server " + hbi.metaEntry.regionServer @@ -857,12 +892,34 @@ Collection ranges = regions.get(key); if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) { for (HbckInfo rng : ranges) { - // TODO offline fix region hole. - errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, "First region should start with an empty key. You need to " + " create a new region and regioninfo in HDFS to plug the hole.", this, rng); + // there is one error, it is FIRST_REGION_STARTKEY_NOT_EMPTY, + // we try to create empty region for this hole. + if (shouldFix() + && (HBaseFsckRepair.checkOnlyOneError(errors.getErrorList(), + ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY)) + && (null != rng)) { + LOG.info("Try to create empty region, start key" + + " is HConstants.EMPTY_BYTE_ARRAY, end key is " + + Bytes.toString(rng.metaEntry.getStartKey()) + + ", Table name is " + + Bytes.toString(rng.metaEntry.getTableName())); + setShouldRerun(); + try { + HBaseFsckRepair.createEmptyRegion(conf, + rng.metaEntry.getTableDesc(), HConstants.EMPTY_BYTE_ARRAY, + rng.metaEntry.getStartKey()); + } catch (IllegalArgumentException e) { + LOG.error(e); + } catch (IOException e) { + LOG.error(e); + } catch (InterruptedException e) { + LOG.error(e); + } + } } } @@ -932,6 +989,27 @@ + Bytes.toStringBinary(holeStopKey) + ". You need to create a new regioninfo and region " + "dir in hdfs to plug the hole."); + // If there is one error that it is HOLE_IN_REGION_CHAIN, + // create empty region to fix this error. + if (shouldFix() + && (HBaseFsckRepair.checkOnlyOneError(errors.getErrorList(), + ERROR_CODE.HOLE_IN_REGION_CHAIN))) { + LOG.info("Trying to create a new regioninfo and region dir " + + "in hdfs to plug the hole, table name is " + getName() + + ", start key is " + Bytes.toStringBinary(key) + + ", end key is " + Bytes.toStringBinary(holeStopKey)); + setShouldRerun(); + try { + HBaseFsckRepair.createEmptyRegion(conf, getName(), key, + holeStopKey); + } catch (IllegalArgumentException e) { + LOG.error(e); + } catch (IOException e) { + LOG.error(e); + } catch (InterruptedException e) { + LOG.error(e); + } + } } } prevKey = key; Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java revision 1302529) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (working copy) @@ -20,18 +20,28 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.zookeeper.KeeperException; @@ -80,6 +90,206 @@ forceOfflineInZK(conf, actualRegion); } + public static void createEmptyRegion(Configuration conf, + HTableDescriptor tblDes, byte[] startKey, byte[] endKey) + throws IllegalArgumentException, IOException, InterruptedException { + HRegionInfo hri = new HRegionInfo(tblDes, startKey, endKey); + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + + HTable metatable = null; + HBaseAdmin admin = null; + try { + metatable = getMetaTable(conf); + metatable.put(put); + admin = getHBaseAdmin(conf); + // flush .META. data to hdfs. + admin.flush(metatable.getTableName()); + admin.assign(hri.getRegionName(), true); + } finally { + if (null != metatable) { + metatable.close(); + } + if (null != admin) { + admin.close(); + } + } + } + + /** + * create empty region + * @param conf + * @param tableName + * @param startKey + * @param endKey + * @throws IllegalArgumentException + * @throws IOException + * @throws InterruptedException + */ + public static void createEmptyRegion(Configuration conf, String tableName, + byte[] startKey, byte[] endKey) throws IllegalArgumentException, + IOException, InterruptedException { + createEmptyRegion(conf, new HTable(tableName).getTableDescriptor(), + startKey, endKey); + } + + /** + * get regionInfo from .META. + * @param conf + * @return + * @throws IOException + */ + public static List getkMetaInfo(Configuration conf) + throws IOException { + List metaList = new ArrayList(); + HTable metatable = null; + ResultScanner scanner = null; + try { + metatable = getMetaTable(conf); + Scan scan = new Scan(); + scanner = metatable.getScanner(scan); + Result result = null; + HRegionInfo hri = null; + while ((result = scanner.next()) != null) { + byte[] bytes = result.getValue(HConstants.CATALOG_FAMILY, + HConstants.REGIONINFO_QUALIFIER); + if (bytes == null || 0 == bytes.length) { + continue; + } + hri = Writables.getHRegionInfo(bytes); + if (null == hri) { + continue; + } + metaList.add(hri); + } + } finally { + if (null != scanner) { + scanner.close(); + } + if (null != metatable) { + metatable.close(); + } + } + return metaList; + } + + /** + * check .META. table hole + * @param metaList + * @param checkHRI + * @return true need to fix .META. table + * @throws IOException + */ + public static boolean checkMetaTableHole(List metaList, + HRegionInfo checkHRI) throws IOException { + if (null == checkHRI) { + return false; + } + List tableList = new ArrayList(); + byte[] tableName = checkHRI.getTableName(); + for (HRegionInfo hri : metaList) { + if ((null != hri) && (Bytes.equals(hri.getTableName(), tableName))) { + tableList.add(hri); + } + } + + // the table is not in .META. + if (0 == tableList.size()) { + return false; + } else { + // check first region + HRegionInfo firstHRI = tableList.get(0); + if (!Bytes.equals(HConstants.EMPTY_BYTE_ARRAY, firstHRI.getStartKey()) + && Bytes.compareTo(checkHRI.getEndKey(), firstHRI.getStartKey()) <= 0) { + return true; + } + + // check last region + HRegionInfo lastHRI = tableList.get(tableList.size() - 1); + if (!Bytes.equals(HConstants.EMPTY_BYTE_ARRAY, lastHRI.getEndKey()) + && Bytes.compareTo(checkHRI.getStartKey(), lastHRI.getEndKey()) >= 0) { + return true; + } + + HRegionInfo oldHRI = null; + for (HRegionInfo hri : tableList) { + if (null != oldHRI) { + if (oldHRI.isOffline() + && Bytes.equals(oldHRI.getStartKey(), hri.getStartKey())) { + // Presume offlined parent + } else if (Bytes.equals(oldHRI.getEndKey(), hri.getStartKey())) { + // Start key of next matches end key of previous + } else { + // there is a hole, and the checkHRI in this hole. + if (Bytes.compareTo(checkHRI.getStartKey(), oldHRI.getEndKey()) <= 0 + && Bytes.compareTo(checkHRI.getEndKey(), oldHRI.getEndKey()) > 0) { + return true; + } else if (Bytes.compareTo(checkHRI.getEndKey(), hri.getStartKey()) >= 0 + && Bytes.compareTo(checkHRI.getStartKey(), hri.getStartKey()) < 0) { + return true; + } else if (Bytes.compareTo(checkHRI.getStartKey(), + oldHRI.getEndKey()) > 0 + && Bytes.compareTo(checkHRI.getEndKey(), hri.getStartKey()) < 0) { + return true; + } + } + } + oldHRI = hri; + } + } + return false; + } + + /** + * fixup meta table hole. + * @param conf + * @param hri + * @throws IOException + */ + public static void fixupMetaTableHole(Configuration conf, HRegionInfo hri) + throws IOException { + Put p = new Put(hri.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + HTable metatable = null; + try { + metatable = getMetaTable(conf); + metatable.put(p); + } finally { + if (null != metatable) { + metatable.close(); + } + } + } + + public static HTable getMetaTable(Configuration conf) throws IOException { + return new HTable(conf, HConstants.META_TABLE_NAME); + } + + public static HBaseAdmin getHBaseAdmin(Configuration conf) + throws MasterNotRunningException, ZooKeeperConnectionException { + return new HBaseAdmin(conf); + } + + /** + * Check one error in errorList. + * @param errorList + * @param findError + * @return + */ + public static boolean checkOnlyOneError(ArrayList errorList, + ERROR_CODE findError) { + boolean isFlag = true; + for (ERROR_CODE error : errorList) { + if (!findError.equals(error)) { + isFlag = false; + break; + } + } + return isFlag; + } + private static void forceOfflineInZK(Configuration conf, final HRegionInfo region) throws ZooKeeperConnectionException, KeeperException, IOException { HConnectionManager.execute(new HConnectable(conf) { Index: src/main/java/org/apache/hadoop/hbase/util/Merge.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/Merge.java (revision 1302529) +++ src/main/java/org/apache/hadoop/hbase/util/Merge.java (working copy) @@ -381,6 +381,12 @@ LOG.error("exiting due to error", e); status = -1; } + + if (0 == status) { + LOG.info("Success to merge the regions."); + } else { + LOG.error("Failed to merge the regions."); + } System.exit(status); } }