From 08c1d898e4394a1abade8cb8c6d3759df25b582a Mon Sep 17 00:00:00 2001 From: Esteban Gutierrez Date: Tue, 7 Mar 2017 01:00:48 -0800 Subject: [PATCH] HBASE-15941 HBCK repair should not unsplit healthy splitted region --- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 153 ++++++++++++++++++++- .../apache/hadoop/hbase/util/HBaseFsckRepair.java | 9 ++ .../util/hbck/TableIntegrityErrorHandler.java | 8 ++ .../hadoop/hbase/util/TestHBaseFsckReplicas.java | 76 ++++++++++ 4 files changed, 243 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 4d44187125..da873c4ab6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -248,6 +248,7 @@ public class HBaseFsck extends Configured implements Closeable { private boolean fixTableOrphans = false; // fix fs holes (missing .tableinfo) private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs private boolean fixSplitParents = false; // fix lingering split parents + private boolean removeParents = false; // remove split parents private boolean fixReferenceFiles = false; // fix lingering reference store file private boolean fixHFileLinks = false; // fix lingering HFileLinks private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows @@ -1108,6 +1109,8 @@ public class HBaseFsck extends Configured implements Closeable { setShouldRerun(); success = fs.rename(path, dst); + debugLsr(dst); + } if (!success) { LOG.error("Failed to sideline reference file " + path); @@ -2488,7 +2491,8 @@ public class HBaseFsck extends Configured implements Closeable { } errors.reportError(ERROR_CODE.LINGERING_SPLIT_PARENT, "Region " + descriptiveName + " is a split parent in META, in HDFS, " - + "and not deployed on any region server. This could be transient."); + + "and not deployed on any region server. This could be transient, " + + "consider to run the catalog janitor first!"); if (shouldFixSplitParents()) { setShouldRerun(); resetSplitParent(hbi); @@ -2885,6 +2889,18 @@ public class HBaseFsck extends Configured implements Closeable { } @Override + public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{ + byte[] key = r1.getStartKey(); + // dup start key + errors.reportError(ERROR_CODE.DUPE_ENDKEYS, + "Multiple regions have the same regionID: " + + Bytes.toStringBinary(key), getTableInfo(), r1); + errors.reportError(ERROR_CODE.DUPE_ENDKEYS, + "Multiple regions have the same regionID: " + + Bytes.toStringBinary(key), getTableInfo(), r2); + } + + @Override public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{ errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN, "There is an overlap in the region chain.", @@ -3018,10 +3034,124 @@ public class HBaseFsck extends Configured implements Closeable { } return; } - + if (shouldRemoveParents()) { + removeParentsAndFixSplits(overlap); + } mergeOverlaps(overlap); } + void removeParentsAndFixSplits(Collection overlap) throws IOException { + Pair range = null; + HbckInfo parent = null; + HbckInfo daughterA = null; + HbckInfo daughterB = null; + Collection daughters = new ArrayList(overlap); + + String thread = Thread.currentThread().getName(); + LOG.info("== [" + thread + "] Attempting fix splits in overlap state."); + + // we only can handle a single split per group at the time + if (overlap.size() > 3) { + LOG.info("Too many overlaps were found on this group, falling back to regular merge."); + return; + } + + for (HbckInfo hi : overlap) { + if (range == null) { + range = new Pair(hi.getStartKey(), hi.getEndKey()); + } else { + if (RegionSplitCalculator.BYTES_COMPARATOR + .compare(hi.getStartKey(), range.getFirst()) < 0) { + range.setFirst(hi.getStartKey()); + } + if (RegionSplitCalculator.BYTES_COMPARATOR + .compare(hi.getEndKey(), range.getSecond()) > 0) { + range.setSecond(hi.getEndKey()); + } + } + } + + LOG.info("This group range is [" + Bytes.toStringBinary(range.getFirst()) + ", " + + Bytes.toStringBinary(range.getSecond()) + "]"); + + // attempt to find a possible parent for the edge case of a split + for (HbckInfo hi : overlap) { + if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0 + && Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) { + LOG.info("This is a parent for this group: " + hi.toString()); + parent = hi; + } + } + + // Remove parent regions from daughters collection + if (parent != null) { + daughters.remove(parent); + } + + // Lets verify that daughters share the regionID at split time and they + // were created after the parent + for (HbckInfo hi : daughters) { + if (Bytes.compareTo(hi.getHdfsHRI().getStartKey(), range.getFirst()) == 0) { + if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) { + daughterA = hi; + } + } + if (Bytes.compareTo(hi.getHdfsHRI().getEndKey(), range.getSecond()) == 0) { + if (parent.getHdfsHRI().getRegionId() < hi.getHdfsHRI().getRegionId()) { + daughterB = hi; + } + } + } + + // daughters must share the same regionID and we should have a parent too + if (daughterA.getHdfsHRI().getRegionId() != daughterB.getHdfsHRI().getRegionId() || parent == null) + return; + + FileSystem fs = FileSystem.get(conf); + LOG.info("Found parent: " + parent.getRegionNameAsString()); + LOG.info("Found potential daughter a: " + daughterA.getRegionNameAsString()); + LOG.info("Found potential daughter b: " + daughterB.getRegionNameAsString()); + LOG.info("Trying to fix parent in overlap by removing the parent."); + try { + closeRegion(parent); + } catch (IOException ioe) { + LOG.warn("Parent region could not be closed, continuing with regular merge...", ioe); + return; + } catch (InterruptedException ie) { + LOG.warn("Parent region could not be closed, continuing with regular merge...", ie); + return; + } + + try { + offline(parent.getRegionName()); + } catch (IOException ioe) { + LOG.warn("Unable to offline parent region: " + parent.getRegionNameAsString() + + ". Just continuing with regular merge... ", ioe); + return; + } + + try { + HBaseFsckRepair.removeParentInMeta(conf, parent.getHdfsHRI()); + } catch (IOException ioe) { + LOG.warn("Unable to remove parent region in META: " + parent.getRegionNameAsString() + + ". Just continuing with regular merge... ", ioe); + return; + } + + sidelineRegionDir(fs, parent); + LOG.info("[" + thread + "] Sidelined parent region dir "+ parent.getHdfsRegionDir() + " into " + + getSidelineDir()); + debugLsr(parent.getHdfsRegionDir()); + + // Make sure we don't have the parents and daughters around + overlap.remove(parent); + overlap.remove(daughterA); + overlap.remove(daughterB); + + LOG.info("Done fixing split."); + + } + void mergeOverlaps(Collection overlap) throws IOException { String thread = Thread.currentThread().getName(); @@ -3207,8 +3337,13 @@ public class HBaseFsck extends Configured implements Closeable { subRange.remove(r1); for (HbckInfo r2 : subRange) { if (r2.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) continue; + // general case of same start key if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) { handler.handleDuplicateStartKeys(r1,r2); + } else if (Bytes.compareTo(r1.getEndKey(), r2.getStartKey())==0 && + r1.getHdfsHRI().getRegionId() == r2.getHdfsHRI().getRegionId()) { + LOG.info("this is a split, log to splits"); + handler.handleSplit(r1, r2); } else { // overlap handler.handleOverlapInRegionChain(r1, r2); @@ -3949,7 +4084,7 @@ public class HBaseFsck extends Configured implements Closeable { HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE, LINGERING_HFILELINK, WRONG_USAGE, EMPTY_META_CELL, EXPIRED_TABLE_LOCK, BOUNDARIES_ERROR, - ORPHAN_TABLE_STATE, NO_TABLE_STATE, UNDELETED_REPLICATION_QUEUE + ORPHAN_TABLE_STATE, NO_TABLE_STATE, UNDELETED_REPLICATION_QUEUE, DUPE_ENDKEYS } void clear(); void report(String message); @@ -4492,10 +4627,19 @@ public class HBaseFsck extends Configured implements Closeable { fixAny |= shouldFix; } + public void setRemoveParents(boolean shouldFix) { + removeParents = shouldFix; + fixAny |= shouldFix; + } + boolean shouldFixSplitParents() { return fixSplitParents; } + boolean shouldRemoveParents() { + return removeParents; + } + public void setFixReferenceFiles(boolean shouldFix) { fixReferenceFiles = shouldFix; fixAny |= shouldFix; @@ -4628,6 +4772,7 @@ public class HBaseFsck extends Configured implements Closeable { out.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps"); out.println(" -maxOverlapsToSideline When fixing region overlaps, allow at most regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)"); out.println(" -fixSplitParents Try to force offline split parents to be online."); + out.println(" -removeParents Try to offline and sideline lingering parents and keep daughter regions."); out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check"); out.println(" -fixReferenceFiles Try to offline lingering reference store files"); out.println(" -fixHFileLinks Try to offline lingering HFileLinks"); @@ -4761,6 +4906,8 @@ public class HBaseFsck extends Configured implements Closeable { setSidelineBigOverlaps(true); } else if (cmd.equals("-fixSplitParents")) { setFixSplitParents(true); + } else if (cmd.equals("-removeParents")) { + setRemoveParents(true); } else if (cmd.equals("-ignorePreCheckPermission")) { setIgnorePreCheckPermission(true); } else if (cmd.equals("-checkCorruptHFiles")) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java index eaf8d545f8..aa96776768 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java @@ -197,4 +197,13 @@ public class HBaseFsckRepair { region.close(); return region; } + + /* + * Remove parent + */ + public static void removeParentInMeta(Configuration conf, HRegionInfo hri) throws IOException { + Connection conn = ConnectionFactory.createConnection(conf); + Table meta = conn.getTable(TableName.META_TABLE_NAME); + MetaTableAccessor.deleteRegion(conn, hri); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java index 4310bf866a..4ca0e7441a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/TableIntegrityErrorHandler.java @@ -74,6 +74,14 @@ public interface TableIntegrityErrorHandler { void handleDuplicateStartKeys(HbckInfo hi1, HbckInfo hi2) throws IOException; /** + * Callback for handling two regions that have the same regionID + * a specific case of a split + * @param hi1 one of the overlapping HbckInfo + * @param hi2 the other overlapping HbckInfo + */ + void handleSplit(HbckInfo hi1, HbckInfo hi2) throws IOException; + + /** * Callback for handling two reigons that overlap in some arbitrary way. * This is a specific case of region overlap, and called for each possible * pair. If two regions have the same start key, the handleDuplicateStartKeys diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java index 9b92a69da4..1744d0b90a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplicas.java @@ -27,8 +27,12 @@ import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.HRegionLocation; + import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.junit.AfterClass; @@ -271,4 +275,76 @@ public class TestHBaseFsckReplicas extends BaseTestHBaseFsck { } } + /** + * Creates and fixes a bad table with a successful split that have a deployed + * start and end keys and region replicas enabled + */ + @Test (timeout=180000) + public void testSplitAndDupeRegionWithRegionReplica() throws Exception { + TableName table = + TableName.valueOf("testSplitAndDupeRegionWithRegionReplica"); + Table meta = null; + + try { + //setupTable(table); + setupTableWithRegionReplica(table, 2); + + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length, countRows()); + + // No Catalog Janitor running + admin.enableCatalogJanitor(false); + meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); + HRegionLocation loc = this.connection.getRegionLocation(table, SPLITS[0], false); + HRegionInfo hriParent = loc.getRegionInfo(); + + // Split Region A just before B + this.connection.getAdmin().split(table, Bytes.toBytes("A@")); + Thread.sleep(1000); + + // We need to make sure the parent region is not in a split state, so we put it in CLOSED state. + regionStates.updateRegionState(hriParent, RegionState.State.CLOSED); + TEST_UTIL.assignRegion(hriParent); + MetaTableAccessor.addRegionToMeta(meta, hriParent); + ServerName server = regionStates.getRegionServerOfRegion(hriParent); + + if (server != null) + TEST_UTIL.assertRegionOnServer(hriParent, server, REGION_ONLINE_TIMEOUT); + + while (findDeployedHSI(getDeployedHRIs((HBaseAdmin) admin), hriParent) == null) { + Thread.sleep(250); + } + + LOG.debug("Finished assignment of parent region"); + + // TODO why is dupe region different from dupe start keys? + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new HBaseFsck.ErrorReporter.ERROR_CODE[] { HBaseFsck.ErrorReporter.ERROR_CODE.NOT_DEPLOYED, + HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, + HBaseFsck.ErrorReporter.ERROR_CODE.DUPE_STARTKEYS, HBaseFsck.ErrorReporter.ERROR_CODE.OVERLAP_IN_REGION_CHAIN}); + assertEquals(3, hbck.getOverlapGroups(table).size()); + // assertEquals(ROWKEYS.length, countRows()); // seems like the "bigger" region won. + + // fix the degenerate region. + hbck = new HBaseFsck(conf, hbfsckExecutorService); + hbck.setDisplayFullReport(); // i.e. -details + hbck.setTimeLag(0); + hbck.setFixHdfsOverlaps(true); + hbck.setRemoveParents(true); + hbck.setFixReferenceFiles(true); + hbck.setFixHFileLinks(true); + hbck.connect(); + hbck.onlineHbck(); + hbck.close(); + + hbck = doFsck(conf, false); + + assertNoErrors(hbck); + assertEquals(0, hbck.getOverlapGroups(table).size()); + assertEquals(ROWKEYS.length, countRows()); + } finally { + cleanupTable(table); + } + } + } -- 2.12.0