From 6718da4049e15f762bd22566c3a4f08d43ec0b84 Mon Sep 17 00:00:00 2001 From: Esteban Gutierrez Date: Tue, 21 Mar 2017 12:02:56 -0700 Subject: [PATCH] HBASE-17799 HBCK region boundaries check can return false negatives when IOExceptions are thrown --- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 162 ++++++++++----------- 1 file changed, 77 insertions(+), 85 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 4eab62b7d9..9d84938669 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -808,102 +808,94 @@ public class HBaseFsck extends Configured implements Closeable { } } - private static class RegionBoundariesInformation { - public byte [] regionName; - public byte [] metaFirstKey; - public byte [] metaLastKey; - public byte [] storesFirstKey; - public byte [] storesLastKey; - @Override - public String toString () { - return "regionName=" + Bytes.toStringBinary(regionName) + - "\nmetaFirstKey=" + Bytes.toStringBinary(metaFirstKey) + - "\nmetaLastKey=" + Bytes.toStringBinary(metaLastKey) + - "\nstoresFirstKey=" + Bytes.toStringBinary(storesFirstKey) + - "\nstoresLastKey=" + Bytes.toStringBinary(storesLastKey); - } - } + public void checkRegionBoundaries() throws IOException { + ByteArrayComparator comparator = new ByteArrayComparator(); + LOG.info("Starting region boundaries check. It might take a while..."); + List regions = MetaTableAccessor.getAllRegions(connection, true); - public void checkRegionBoundaries() { - try { - ByteArrayComparator comparator = new ByteArrayComparator(); - List regions = MetaTableAccessor.getAllRegions(connection, true); - final RegionBoundariesInformation currentRegionBoundariesInformation = - new RegionBoundariesInformation(); - Path hbaseRoot = FSUtils.getRootDir(getConf()); - for (HRegionInfo regionInfo : regions) { - Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); - currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); - // For each region, get the start and stop key from the META and compare them to the - // same information from the Stores. - Path path = new Path(tableDir, regionInfo.getEncodedName()); - FileSystem fs = path.getFileSystem(getConf()); - FileStatus[] files = fs.listStatus(path); - // For all the column families in this region... - byte[] storeFirstKey = null; - byte[] storeLastKey = null; - for (FileStatus file : files) { - String fileName = file.getPath().toString(); - fileName = fileName.substring(fileName.lastIndexOf("/") + 1); - if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { - FileStatus[] storeFiles = fs.listStatus(file.getPath()); + Path hbaseRoot = FSUtils.getRootDir(getConf()); + int scanned = 0; + LOG.info("Scanning " + regions.size() + " (online) regions for integrity of store files and META boundaries."); + for (HRegionInfo regionInfo : regions) { + Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); + // For each region, get the start and stop key from the META and compare them to the + // same information from the Stores. + Path path = new Path(tableDir, regionInfo.getEncodedName()); + FileStatus[] files = null; + FileSystem fs = path.getFileSystem(getConf()); + try { + files = fs.listStatus(path); + } catch (IOException e) { + LOG.warn("Couldn't list content of region directory: " + path + " skipping.", e); + continue; + } + // For all the column families in this region... + byte[] storeFirstKey = null; + byte[] storeLastKey = null; + for (FileStatus file : files) { + String fileName = file.getPath().toString(); + fileName = fileName.substring(fileName.lastIndexOf("/") + 1); + if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { + FileStatus[] storeFiles; + try { + storeFiles = fs.listStatus(file.getPath()); + } catch (IOException e) { + LOG.warn("Skipping region " + regionInfo.getRegionNameAsString() + + " since region boundaries can't be calculated correctly, got: ", e); + break; + } + + if (storeFiles.length > 0) { // For all the stores in this column family. for (FileStatus storeFile : storeFiles) { - HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( - getConf()), getConf()); - if ((reader.getFirstKey() != null) - && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, - ((KeyValue.KeyOnlyKeyValue) reader.getFirstKey()).getKey()) > 0))) { - storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey()).getKey(); + Path hfilePath = null; + if (HFileLink.isHFileLink(storeFile.getPath())) { + LOG.warn("Skipping HFileLink file: " + storeFile.getPath()); + break; + } else { + hfilePath = storeFile.getPath(); } - if ((reader.getLastKey() != null) - && ((storeLastKey == null) || (comparator.compare(storeLastKey, - ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey())) < 0)) { + // Lets try to read the file finally + try { + HFile.Reader reader = HFile.createReader(fs, hfilePath, new CacheConfig(getConf()), getConf()); + storeFirstKey = ((KeyValue.KeyOnlyKeyValue)reader.getFirstKey()).getKey(); storeLastKey = ((KeyValue.KeyOnlyKeyValue)reader.getLastKey()).getKey(); + reader.close(); + scanned++; + + // For a region to be correct, we need the META start key to be smaller or equal to the + // smallest start key from all the stores, and the start key from the next META entry to + // be bigger than the last key from all the current stores. First region start key is null; + // Last region end key is null; some regions can be empty and not have any store. + if (regionInfo.containsRange(keyOnly(storeFirstKey), keyOnly(storeLastKey))) { + LOG.debug(hfilePath + " is within boundaries:\n" + + "store: [" + Bytes.toStringBinary(keyOnly(storeFirstKey)) + + " -> " + Bytes.toStringBinary(keyOnly(storeLastKey)) + ") " + + "region: [" + Bytes.toStringBinary(regionInfo.getStartKey()) + + " -> " + Bytes.toStringBinary(regionInfo.getEndKey()) + ")"); + } else { + errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, + "Region boundaries mismatch in " + regionInfo.getRegionNameAsString() + " and store " + + hfilePath, tablesInfo.get(regionInfo.getTable())); + LOG.warn(hfilePath + " is not within boundaries:\n" + + "store: [" + Bytes.toStringBinary(keyOnly(storeFirstKey)) + + " -> " + Bytes.toStringBinary(keyOnly(storeLastKey)) + ") " + + "region: [" + Bytes.toStringBinary(regionInfo.getStartKey()) + + " -> " + Bytes.toStringBinary(regionInfo.getEndKey()) + ")"); + } + } catch (IOException e) { + LOG.warn("Skipping " + hfilePath + " from region " + + regionInfo.getRegionNameAsString() + " got an exception.", e); } - reader.close(); } + } else { + // empty regions are usually harmless. + LOG.debug("Region directory " + regionInfo.getRegionNameAsString() + " is empty."); } } - currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); - currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); - currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); - currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); - if (currentRegionBoundariesInformation.metaFirstKey.length == 0) - currentRegionBoundariesInformation.metaFirstKey = null; - if (currentRegionBoundariesInformation.metaLastKey.length == 0) - currentRegionBoundariesInformation.metaLastKey = null; - - // For a region to be correct, we need the META start key to be smaller or equal to the - // smallest start key from all the stores, and the start key from the next META entry to - // be bigger than the last key from all the current stores. First region start key is null; - // Last region end key is null; some regions can be empty and not have any store. - - boolean valid = true; - // Checking start key. - if ((currentRegionBoundariesInformation.storesFirstKey != null) - && (currentRegionBoundariesInformation.metaFirstKey != null)) { - valid = valid - && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, - currentRegionBoundariesInformation.metaFirstKey) >= 0; - } - // Checking stop key. - if ((currentRegionBoundariesInformation.storesLastKey != null) - && (currentRegionBoundariesInformation.metaLastKey != null)) { - valid = valid - && comparator.compare(currentRegionBoundariesInformation.storesLastKey, - currentRegionBoundariesInformation.metaLastKey) < 0; - } - if (!valid) { - errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", - tablesInfo.get(regionInfo.getTable())); - LOG.warn("Region's boundaries not aligned between stores and META for:"); - LOG.warn(currentRegionBoundariesInformation); - } } - } catch (IOException e) { - LOG.error(e); } + LOG.info("Region boundaries test scanned: " + scanned + " files in " + regions.size() +" regions."); } /** -- 2.12.1