diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 0d69159..c7bbab8 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -210,7 +210,7 @@ public class HBaseFsck extends Configured implements Closeable { private Table meta; // threads to do ||izable tasks: retrieve data from regionservers, handle overlapping regions protected ExecutorService executor; - private long startMillis = System.currentTimeMillis(); + private long startMillis = EnvironmentEdgeManager.currentTime(); private HFileCorruptionChecker hfcc; private int retcode = 0; private Path HBCK_LOCK_PATH; @@ -293,6 +293,7 @@ public class HBaseFsck extends Configured implements Closeable { new HashMap(); private final RetryCounterFactory lockFileRetryCounterFactory; + private Map> skippedRegions = new HashMap>(); /** * Constructor @@ -1716,7 +1717,7 @@ public class HBaseFsck extends Configured implements Closeable { return false; } ServerName sn = metaLocation.getServerName(); - MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis()); + MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime()); HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName()); if (hbckInfo == null) { regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m)); @@ -1861,11 +1862,32 @@ public class HBaseFsck extends Configured implements Closeable { @Override public synchronized Void call() throws Exception { - checkRegionConsistency(key, hbi); + try { + checkRegionConsistency(key, hbi); + } catch (Exception e) { + // If the region is non-META region, skip this region and send warning/error message; if + // the region is META region, we should not continue. + LOG.warn("Unable to complete check or repair the region '" + hbi.getRegionNameAsString() + + "'.", e); + if (hbi.getHdfsHRI().isMetaRegion()) { + throw e; + } + LOG.warn("Skip region '" + hbi.getRegionNameAsString() + "'"); + addSkippedRegion(hbi); + } return null; } } + private void addSkippedRegion(final HbckInfo hbi) { + Set skippedRegionStrings = skippedRegions.get(hbi.getTableName()); + if (skippedRegionStrings == null) { + skippedRegionStrings = new HashSet(); + } + skippedRegionStrings.add(hbi.getRegionNameAsString()); + skippedRegions.put(hbi.getTableName(), skippedRegionStrings); + } + /** * Check and fix table states, assumes full info available: * - tableInfos @@ -2155,7 +2177,7 @@ public class HBaseFsck extends Configured implements Closeable { inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline(); boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable()); boolean recentlyModified = inHdfs && - hbi.getModTime() + timelag > System.currentTimeMillis(); + hbi.getModTime() + timelag > EnvironmentEdgeManager.currentTime(); // ========== First the healthy cases ============= if (hbi.containsOnlyHdfsEdits()) { @@ -3160,7 +3182,7 @@ public class HBaseFsck extends Configured implements Closeable { */ HTableDescriptor[] getTables(AtomicInteger numSkipped) { List tableNames = new ArrayList(); - long now = System.currentTimeMillis(); + long now = EnvironmentEdgeManager.currentTime(); for (HbckInfo hbi : regionInfoMap.values()) { MetaEntry info = hbi.metaEntry; @@ -3701,9 +3723,18 @@ public class HBaseFsck extends Configured implements Closeable { if (errors.tableHasErrors(tInfo)) { errors.print("Table " + tInfo.getName() + " is inconsistent."); } else { - errors.print(" " + tInfo.getName() + " is okay."); + errors.print("Table " + tInfo.getName() + " is okay" + + (skippedRegions.containsKey(tInfo.getName())? " (with some skipped regions)." : ".")); } errors.print(" Number of regions: " + tInfo.getNumRegions()); + if (skippedRegions.containsKey(tInfo.getName())) { + Set skippedRegionStrings = skippedRegions.get(tInfo.getName()); + System.out.println(" Number of skipped regions: " + skippedRegionStrings.size()); + System.out.println(" List of skipped regions:"); + for(String sr : skippedRegionStrings) { + System.out.println(" " + sr); + } + } sb.setLength(0); // clear out existing buffer, if any. sb.append(" Deployed on: "); for (ServerName server : tInfo.deployedOn) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java index 4c742e3..7de7af8 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; @@ -35,7 +34,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.master.RegionState; @@ -123,8 +121,8 @@ public class HBaseFsckRepair { public static void waitUntilAssigned(Admin admin, HRegionInfo region) throws IOException, InterruptedException { long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000); - long expiration = timeout + System.currentTimeMillis(); - while (System.currentTimeMillis() < expiration) { + long expiration = timeout + EnvironmentEdgeManager.currentTime(); + while (EnvironmentEdgeManager.currentTime() < expiration) { try { Map rits= admin.getClusterStatus().getRegionsInTransition(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 17dab15..dab2508 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -103,7 +103,6 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.SplitTransactionFactory; import org.apache.hadoop.hbase.regionserver.SplitTransactionImpl; import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction; -import org.apache.hadoop.hbase.security.access.AccessControlClient; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;