From ede7b03c350c9b09647085aa1ade136766bac78f Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Wed, 8 Apr 2015 16:22:22 -0400 Subject: [PATCH] HBASE-12987 Pare repeated hbck output and increase verbosity in long-running tasks. --- .../java/org/apache/hadoop/hbase/util/FSUtils.java | 55 +++++++++++++++++++++- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 38 ++++++++++++--- 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index d72634e..57eeb7c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionPlacementMaintainer; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.security.AccessDeniedException; +import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.FSProtos; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -1520,6 +1521,28 @@ public abstract class FSUtils { public static Map getTableStoreFilePathMap(Map map, final FileSystem fs, final Path hbaseRootDir, TableName tableName) throws IOException { + return getTableStoreFilePathMap(map, fs, hbaseRootDir, tableName, null); + } + + /** + * Runs through the HBase rootdir/tablename and creates a reverse lookup map for + * table StoreFile names to the full Path. + *
+ * Example...
+ * Key = 3944417774205889744
+ * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744 + * + * @param map map to add values. If null, this method will create and populate one to return + * @param fs The file system to use. + * @param hbaseRootDir The root directory to scan. + * @param tableName name of the table to scan. + * @param errors ErrorReporter instance or null + * @return Map keyed by StoreFile name with a value of the full Path. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableStoreFilePathMap(Map map, + final FileSystem fs, final Path hbaseRootDir, TableName tableName, ErrorReporter errors) + throws IOException { if (map == null) { map = new HashMap(); } @@ -1531,15 +1554,24 @@ public abstract class FSUtils { PathFilter familyFilter = new FamilyDirFilter(fs); FileStatus[] regionDirs = fs.listStatus(tableDir, new RegionDirFilter(fs)); for (FileStatus regionDir : regionDirs) { + if (null != errors) { + errors.progress(); + } Path dd = regionDir.getPath(); // else its a region name, now look in region for families FileStatus[] familyDirs = fs.listStatus(dd, familyFilter); for (FileStatus familyDir : familyDirs) { + if (null != errors) { + errors.progress(); + } Path family = familyDir.getPath(); // now in family, iterate over the StoreFiles and // put in map FileStatus[] familyStatus = fs.listStatus(family); for (FileStatus sfStatus : familyStatus) { + if (null != errors) { + errors.progress(); + } Path sf = sfStatus.getPath(); map.put( sf.getName(), sf); } @@ -1560,7 +1592,6 @@ public abstract class FSUtils { return result; } - /** * Runs through the HBase rootdir and creates a reverse lookup map for * table StoreFile names to the full Path. @@ -1577,6 +1608,26 @@ public abstract class FSUtils { public static Map getTableStoreFilePathMap( final FileSystem fs, final Path hbaseRootDir) throws IOException { + return getTableStoreFilePathMap(fs, hbaseRootDir, null); + } + + /** + * Runs through the HBase rootdir and creates a reverse lookup map for + * table StoreFile names to the full Path. + *
+ * Example...
+ * Key = 3944417774205889744
+ * Value = hdfs://localhost:51169/user/userid/-ROOT-/70236052/info/3944417774205889744 + * + * @param fs The file system to use. + * @param hbaseRootDir The root directory to scan. + * @param errors ErrorReporter instance or null + * @return Map keyed by StoreFile name with a value of the full Path. + * @throws IOException When scanning the directory fails. + */ + public static Map getTableStoreFilePathMap( + final FileSystem fs, final Path hbaseRootDir, ErrorReporter errors) + throws IOException { Map map = new HashMap(); // if this method looks similar to 'getTableFragmentation' that is because @@ -1585,7 +1636,7 @@ public abstract class FSUtils { // only include the directory paths to tables for (Path tableDir : FSUtils.getTableDirs(fs, hbaseRootDir)) { getTableStoreFilePathMap(map, fs, hbaseRootDir, - FSUtils.getTableName(tableDir)); + FSUtils.getTableName(tableDir), errors); } return map; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index cad0ba2..eef9dac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -550,7 +550,9 @@ public class HBaseFsck extends Configured { // load regiondirs and regioninfos from HDFS if (shouldCheckHdfs()) { + LOG.info("Loading region directories from HDFS"); loadHdfsRegionDirs(); + LOG.info("Loading region information from HDFS"); loadHdfsRegionInfos(); } @@ -560,6 +562,8 @@ public class HBaseFsck extends Configured { // fix the orphan tables fixOrphanTables(); + LOG.info("Checking and fixing region consistency"); + // Check and fix consistency checkAndFixConsistency(); @@ -875,7 +879,10 @@ public class HBaseFsck extends Configured { Configuration conf = getConf(); Path hbaseRoot = FSUtils.getRootDir(conf); FileSystem fs = hbaseRoot.getFileSystem(conf); - Map allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot); + LOG.info("Computing mapping of all store files"); + Map allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, errors); + errors.print(""); + LOG.info("Validating mapping using HDFS state"); for (Path path: allFiles.values()) { boolean isReference = false; try { @@ -1073,6 +1080,7 @@ public class HBaseFsck extends Configured { } loadTableInfosForTablesWithNoRegion(); + errors.print(""); return tablesInfo; } @@ -1252,6 +1260,7 @@ public class HBaseFsck extends Configured { */ private void suggestFixes( SortedMap tablesInfo) throws IOException { + logParallelMerge(); for (TableInfo tInfo : tablesInfo.values()) { TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors); tInfo.checkRegionChain(handler); @@ -1322,9 +1331,23 @@ public class HBaseFsck extends Configured { return true; } + /** + * Log an appropriate message about whether or not overlapping merges are computed in parallel. + */ + private void logParallelMerge() { + if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) { + LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" + + " false to run serially."); + } else { + LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" + + " true to run in parallel."); + } + } + private SortedMap checkHdfsIntegrity(boolean fixHoles, boolean fixOverlaps) throws IOException { LOG.info("Checking HBase region split map from HDFS data..."); + logParallelMerge(); for (TableInfo tInfo : tablesInfo.values()) { TableIntegrityErrorHandler handler; if (fixHoles || fixOverlaps) { @@ -1566,6 +1589,7 @@ public class HBaseFsck extends Configured { LOG.warn("Could not load region dir " , e.getCause()); } } + errors.print(""); } /** @@ -2160,6 +2184,7 @@ public class HBaseFsck extends Configured { loadTableInfosForTablesWithNoRegion(); + logParallelMerge(); for (TableInfo tInfo : tablesInfo.values()) { TableIntegrityErrorHandler handler = tInfo.new IntegrityFixSuggester(tInfo, errors); if (!tInfo.checkRegionChain(handler)) { @@ -2772,15 +2797,11 @@ public class HBaseFsck extends Configured { // TODO fold this into the TableIntegrityHandler if (getConf().getBoolean("hbasefsck.overlap.merge.parallel", true)) { - LOG.info("Handling overlap merges in parallel. set hbasefsck.overlap.merge.parallel to" + - " false to run serially."); boolean ok = handleOverlapsParallel(handler, prevKey); if (!ok) { return false; } } else { - LOG.info("Handling overlap merges serially. set hbasefsck.overlap.merge.parallel to" + - " true to run in parallel."); for (Collection overlap : overlapGroups.asMap().values()) { handler.handleOverlapGroup(overlap); } @@ -3472,6 +3493,8 @@ public class HBaseFsck extends Configured { static class PrintingErrorReporter implements ErrorReporter { public int errorCount = 0; private int showProgress; + // How frequently calls to progress() will create output + private static final int progressThreshold = 100; Set errorTables = new HashSet(); @@ -3586,7 +3609,7 @@ public class HBaseFsck extends Configured { @Override public synchronized void progress() { - if (showProgress++ == 10) { + if (showProgress++ == progressThreshold) { if (!summary) { System.out.print("."); } @@ -3683,6 +3706,7 @@ public class HBaseFsck extends Configured { // level 2: //* FileStatus[] regionDirs = fs.listStatus(tableDir.getPath()); for (FileStatus regionDir : regionDirs) { + errors.progress(); String encodedName = regionDir.getPath().getName(); // ignore directories that aren't hexadecimal if (!encodedName.toLowerCase().matches("[0-9a-f]+")) { @@ -3710,6 +3734,7 @@ public class HBaseFsck extends Configured { FileStatus[] subDirs = fs.listStatus(regionDir.getPath()); Path ePath = HLogUtil.getRegionDirRecoveredEditsDir(regionDir.getPath()); for (FileStatus subDir : subDirs) { + errors.progress(); String sdName = subDir.getPath().getName(); if (!sdName.startsWith(".") && !sdName.equals(ePath.getName())) { he.hdfsOnlyEdits = false; @@ -3750,6 +3775,7 @@ public class HBaseFsck extends Configured { // only load entries that haven't been loaded yet. if (hbi.getHdfsHRI() == null) { try { + errors.progress(); hbck.loadHdfsRegioninfo(hbi); } catch (IOException ioe) { String msg = "Orphan region in HDFS: Unable to load .regioninfo from table " -- 1.9.5 (Apple Git-50.3)