Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1302260) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy) @@ -552,6 +552,13 @@ if (!foundVersionFile) { errors.reportError(ERROR_CODE.NO_VERSION_FILE, "Version file does not exist in root dir " + rootDir); + if (shouldFix()) { + LOG.info("Trying to create a new " + HConstants.VERSION_FILE_NAME + + " file."); + setShouldRerun(); + FSUtils.setVersion(fs, rootDir, + conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); + } } // level 1: /* @@ -686,10 +693,29 @@ errors.reportError(ERROR_CODE.NOT_IN_META_OR_DEPLOYED, "Region " + descriptiveName + " on HDFS, but not listed in META " + "or deployed on any region server"); + if (shouldFix()) { + setShouldRerun(); + // load region info from hdfs. + if (null == hbi.metaEntry) { + loadMetaEntry(hbi); + } + errors.print("Trying to plug the hole in .META, encodename is " + + hbi.metaEntry.getEncodedName()); + HBaseFsckRepair.fixupMetaTableHole(conf, hbi.metaEntry); + } } else if (!inMeta && inHdfs && isDeployed) { errors.reportError(ERROR_CODE.NOT_IN_META, "Region " + descriptiveName + " not in META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn)); - + if (shouldFix()) { + setShouldRerun(); + // load region info from hdfs. + if (null == hbi.metaEntry) { + loadMetaEntry(hbi); + } + errors.print("Trying to plug the hole in .META, encodename is " + + hbi.metaEntry.getEncodedName()); + HBaseFsckRepair.fixupMetaTableHole(conf, hbi.metaEntry); + } // ========== Cases where the region is in META ============= } else if (inMeta && !inHdfs && !isDeployed) { errors.reportError(ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED, "Region " @@ -712,6 +738,11 @@ errors.reportError(ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, "Region " + descriptiveName + " should not be deployed according " + "to META, but is deployed on " + Joiner.on(", ").join(hbi.deployedOn)); + if (shouldFix()) { + errors.print("Trying to close the region, " + descriptiveName); + setShouldRerun(); + HBaseFsckRepair.closeRegion(this.conf, hbi.metaEntry, hbi.deployedOn); + } } else if (inMeta && inHdfs && isMultiplyDeployed) { errors.reportError(ERROR_CODE.MULTI_DEPLOYED, "Region " + descriptiveName + " is listed in META on region server " + hbi.metaEntry.regionServer @@ -857,12 +888,34 @@ Collection ranges = regions.get(key); if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) { for (HbckInfo rng : ranges) { - // TODO offline fix region hole. - errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, "First region should start with an empty key. You need to " + " create a new region and regioninfo in HDFS to plug the hole.", this, rng); + // there is one error, it is FIRST_REGION_STARTKEY_NOT_EMPTY, + // we try to create empty region for this hole. + if (shouldFix() + && (HBaseFsckRepair.checkOnlyOneError(errors.getErrorList(), + ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY)) + && (null != rng)) { + LOG.info("Try to create empty region, start key" + + " is HConstants.EMPTY_BYTE_ARRAY, end key is " + + Bytes.toString(rng.metaEntry.getStartKey()) + + ", Table name is " + + Bytes.toString(rng.metaEntry.getTableName())); + setShouldRerun(); + try { + HBaseFsckRepair.createEmptyRegion(conf, + rng.metaEntry.getTableDesc(), HConstants.EMPTY_BYTE_ARRAY, + rng.metaEntry.getStartKey()); + } catch (IllegalArgumentException e) { + LOG.error(e); + } catch (IOException e) { + LOG.error(e); + } catch (InterruptedException e) { + LOG.error(e); + } + } } } @@ -932,6 +985,27 @@ + Bytes.toStringBinary(holeStopKey) + ". You need to create a new regioninfo and region " + "dir in hdfs to plug the hole."); + // If there is one error that it is HOLE_IN_REGION_CHAIN, + // create empty region to fix this error. + if (shouldFix() + && (HBaseFsckRepair.checkOnlyOneError(errors.getErrorList(), + ERROR_CODE.HOLE_IN_REGION_CHAIN))) { + LOG.info("Trying to create a new regioninfo and region dir " + + "in hdfs to plug the hole, table name is " + getName() + + ", start key is " + Bytes.toStringBinary(key) + + ", end key is " + Bytes.toStringBinary(holeStopKey)); + setShouldRerun(); + try { + HBaseFsckRepair.createEmptyRegion(conf, getName(), key, + holeStopKey); + } catch (IllegalArgumentException e) { + LOG.error(e); + } catch (IOException e) { + LOG.error(e); + } catch (InterruptedException e) { + LOG.error(e); + } + } } } prevKey = key; Index: src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (revision 1302260) +++ src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java (working copy) @@ -20,18 +20,25 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.zookeeper.KeeperException; @@ -62,7 +69,25 @@ // Force ZK node to OFFLINE so master assigns forceOfflineInZK(conf, actualRegion); } + + /** + * Close region on the servers silently. + * @param conf + * @param region + * @param servers + * @throws IOException + * @throws InterruptedException + */ + public static void closeRegion(Configuration conf, HRegionInfo region, + List servers) throws IOException, InterruptedException { + HRegionInfo actualRegion = new HRegionInfo(region); + // Close region on the servers silently + for (HServerAddress server : servers) { + closeRegionSilentlyAndWait(conf, server, actualRegion); + } + } + /** * Fix unassigned by creating/transition the unassigned ZK node for this * region to OFFLINE state with a special flag to tell the master that this @@ -80,6 +105,81 @@ forceOfflineInZK(conf, actualRegion); } + public static void createEmptyRegion(Configuration conf, + HTableDescriptor tblDes, byte[] startKey, byte[] endKey) + throws IllegalArgumentException, IOException, InterruptedException { + HRegionInfo hri = new HRegionInfo(tblDes, startKey, endKey); + Put put = new Put(hri.getRegionName()); + put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + + HTable metatable = getMetaTable(conf); + metatable.put(put); + HBaseAdmin admin = getHBaseAdmin(conf); + // flush .META. data to hdfs. + admin.flush(metatable.getTableName()); + admin.assign(hri.getRegionName(), true); + } + + /** + * create empty table + * @param conf + * @param tableName + * @param startKey + * @param endKey + * @throws IllegalArgumentException + * @throws IOException + * @throws InterruptedException + */ + public static void createEmptyRegion(Configuration conf, String tableName, + byte[] startKey, byte[] endKey) throws IllegalArgumentException, + IOException, InterruptedException { + createEmptyRegion(conf, new HTable(tableName).getTableDescriptor(), + startKey, endKey); + } + + /** + * fixup meta table hole. + * @param conf + * @param hri + * @throws IOException + */ + public static void fixupMetaTableHole(Configuration conf, HRegionInfo hri) + throws IOException { + Put p = new Put(hri.getRegionName()); + p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + Writables.getBytes(hri)); + HTable metatable = getMetaTable(conf); + metatable.put(p); + } + + public static HTable getMetaTable(Configuration conf) throws IOException { + return new HTable(conf, HConstants.META_TABLE_NAME); + } + + public static HBaseAdmin getHBaseAdmin(Configuration conf) + throws MasterNotRunningException, ZooKeeperConnectionException { + return new HBaseAdmin(conf); + } + + /** + * Check one error in errorList. + * @param errorList + * @param findError + * @return + */ + public static boolean checkOnlyOneError(ArrayList errorList, + ERROR_CODE findError) { + boolean isFlag = true; + for (ERROR_CODE error : errorList) { + if (!findError.equals(error)) { + isFlag = false; + break; + } + } + return isFlag; + } + private static void forceOfflineInZK(Configuration conf, final HRegionInfo region) throws ZooKeeperConnectionException, KeeperException, IOException { HConnectionManager.execute(new HConnectable(conf) { Index: src/main/java/org/apache/hadoop/hbase/util/Merge.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/util/Merge.java (revision 1302260) +++ src/main/java/org/apache/hadoop/hbase/util/Merge.java (working copy) @@ -381,6 +381,12 @@ LOG.error("exiting due to error", e); status = -1; } + + if (0 == status) { + LOG.info("Success to merger the regions."); + } else { + LOG.error("Failed to merger the regions."); + } System.exit(status); } }