Index: hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (revision 1380945) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (working copy) @@ -26,6 +26,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -182,6 +183,7 @@ private boolean fixHdfsHoles = false; // fix fs holes? private boolean fixHdfsOverlaps = false; // fix fs overlaps (risky) private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo) + private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs private boolean fixSplitParents = false; // fix lingering split parents @@ -231,6 +233,8 @@ * When initially looking at HDFS, we attempt to find any orphaned data. */ private List orphanHdfsDirs = Collections.synchronizedList(new ArrayList()); + + private Set orphanTableDirs = new HashSet(); /** * Constructor @@ -333,7 +337,8 @@ */ public void offlineHdfsIntegrityRepair() throws IOException, InterruptedException { // Initial pass to fix orphans. - if (shouldFixHdfsOrphans() || shouldFixHdfsHoles() || shouldFixHdfsOverlaps()) { + if (shouldFixHdfsOrphans() || shouldFixHdfsHoles() + || shouldFixHdfsOverlaps() || shouldFixTableOrphans()) { LOG.info("Loading regioninfos HDFS"); // if nothing is happening this should always complete in two iterations. int maxIterations = conf.getInt("hbase.hbck.integrityrepair.iterations.max", 3); @@ -385,7 +390,7 @@ if (!checkMetaOnly) { reportTablesInFlux(); } - + // get regions according to what is online on each RegionServer loadDeployedRegions(); @@ -399,6 +404,9 @@ // Get disabled tables from ZooKeeper loadDisabledTables(); + // fix the orphan tables + fixOrphanTables(); + // Check and fix consistency checkAndFixConsistency(); @@ -712,9 +720,13 @@ hbaseRoot, tableName); modTInfo.htds.add(htd); } catch (IOException ioe) { - LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe); - errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, - "Unable to read .tableinfo from " + hbaseRoot); + if (!orphanTableDirs.contains(tableName)) { + LOG.warn("Unable to read .tableinfo from " + hbaseRoot, ioe); + //should only report once for each table + errors.reportError(ERROR_CODE.NO_TABLEINFO_FILE, + "Unable to read .tableinfo from " + hbaseRoot + "/" + tableName); + orphanTableDirs.add(tableName); + } } } modTInfo.addRegionInfo(hbi); @@ -722,7 +734,50 @@ return tablesInfo; } + + public void fixOrphanTables() throws IOException { + if (shouldFixTableOrphans() && !orphanTableDirs.isEmpty()) { + Path hbaseRoot = new Path(conf.get(HConstants.HBASE_DIR)); + List tmpList = new ArrayList(); + tmpList.addAll(orphanTableDirs); + HTableDescriptor[] htds = getHTableDescriptors(tmpList); + Iterator iter = orphanTableDirs.iterator(); + int j = 0; + while (iter.hasNext()) { + String tableName = (String) iter.next(); + errors.print("Try to fix orphan table: " + tableName); + if (j < htds.length) { + if (tableName.equals(Bytes.toString(htds[j].getName()))) { + HTableDescriptor htd = htds[j]; + errors.print("fixing table: " + tableName); + FSTableDescriptors.createTableDescriptor( + hbaseRoot.getFileSystem(conf), hbaseRoot, htd, true); + iter.remove(); + j++; + fixes++; + } + } else { + errors.report("Failed to fix orphan table: " + tableName); + } + } + + if (orphanTableDirs.isEmpty()) { + // all orphanTableDirs are luckily recovered + // re-run doFsck after recovering the .tableinfo file + setShouldRerun(); + LOG.warn("Strongly recommend to re-run manually hfsck after all orphanTableDirs being fixed"); + } else { + errors.report("Failed to repair " + orphanTableDirs.size() + + " OrphanTables"); + } + + } + //cleanup the list + orphanTableDirs.clear(); + + } + /** * This borrows code from MasterFileSystem.bootstrap() * @@ -3017,7 +3072,15 @@ boolean shouldFixHdfsHoles() { return fixHdfsHoles; } - + + public void setFixTableOrphans(boolean shouldFix) { + fixTableOrphans = shouldFix; + } + + boolean shouldFixTableOrphans() { + return fixTableOrphans; + } + public void setFixHdfsOverlaps(boolean shouldFix) { fixHdfsOverlaps = shouldFix; } @@ -3159,6 +3222,7 @@ System.err.println(" -fixMeta Try to fix meta problems. This assumes HDFS region info is good."); System.err.println(" -fixHdfsHoles Try to fix region holes in hdfs."); System.err.println(" -fixHdfsOrphans Try to fix region dirs with no .regioninfo file in hdfs"); + System.err.println(" -fixTableOrphans Try to fix table dirs with no .tableinfo file in hdfs (online mode only)"); System.err.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs."); System.err.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs."); System.err.println(" -maxMerge When fixing region overlaps, allow at most regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)"); @@ -3263,6 +3327,8 @@ setFixHdfsHoles(true); } else if (cmd.equals("-fixHdfsOrphans")) { setFixHdfsOrphans(true); + } else if (cmd.equals("-fixTableOrphans")) { + setFixTableOrphans(true); } else if (cmd.equals("-fixHdfsOverlaps")) { setFixHdfsOverlaps(true); } else if (cmd.equals("-fixVersionFile")) { @@ -3389,6 +3455,7 @@ setFixHdfsHoles(false); setFixHdfsOverlaps(false); setFixVersionFile(false); + setFixTableOrphans(false); errors.resetErrors(); code = onlineHbck(); setRetCode(code);