From 1302e163d96351bbf48c2564f9af1a5cfc8aec39 Mon Sep 17 00:00:00 2001 From: Syeda Date: Tue, 12 Mar 2019 18:53:25 +0530 Subject: [PATCH] HBASE-21920 Ignoring 'empty' end_key while calculating end_key for new region in HBCK -fixHdfsOverlaps command can cause data loss --- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 7 +++- .../apache/hadoop/hbase/util/TestHBaseFsck.java | 47 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index d0fa17d..6cb3d20 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -3133,7 +3133,12 @@ public class HBaseFsck extends Configured implements Closeable { .compare(hi.getStartKey(), range.getFirst()) < 0) { range.setFirst(hi.getStartKey()); } - if (RegionSplitCalculator.BYTES_COMPARATOR + if ((RegionSplitCalculator.BYTES_COMPARATOR + .compare(range.getSecond(), HConstants.EMPTY_END_ROW) == 0) + || (RegionSplitCalculator.BYTES_COMPARATOR.compare(hi.getEndKey(), + HConstants.EMPTY_END_ROW) == 0)) { + range.setSecond(HConstants.EMPTY_END_ROW); + } else if (RegionSplitCalculator.BYTES_COMPARATOR .compare(hi.getEndKey(), range.getSecond()) > 0) { range.setSecond(hi.getEndKey()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 917d7d3..9e722f8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -3221,6 +3221,53 @@ public class TestHBaseFsck { } } + /** + * This creates and fixes a bad table where a region is completely contained by another region. + * Verify whether it considers "empty" start-key as minimum and "empty" end-key as maximum while + * creating new region to merge overlap regions. + */ + @Test(timeout = 180000) + public void testStart_keyAndEnd_keyCalcForRegionOverlap() throws Exception { + TableName table = TableName.valueOf("testStart_keyAndEnd_keyCalcForRegionOverlap"); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + try { + HTableDescriptor desc = new HTableDescriptor(table); + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); + desc.addFamily(hcd); // If a table has no CF's it doesn't get checked + createTable(TEST_UTIL, desc, null); + tbl = new HTable(cluster.getConfiguration(), desc.getTableName()); + for (int i = 0; i < 5; i++) { + Put p1 = new Put(Integer.toString(i).getBytes()); + p1.add(FAM, "q1".getBytes(), "v".getBytes()); + tbl.put(p1); + } + admin.flush(desc.getTableName()); + // Mess it up by creating an overlap + HRegionInfo hriOverlap = + createRegion(tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW, Bytes.toBytes("4")); + TEST_UTIL.assignRegion(hriOverlap); + // verify overlaps exists + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS }); + assertEquals(2, hbck.getOverlapGroups(table).size()); + // fix the problem. + doFsck(conf, true); + // verify that overlaps are fixed + HBaseFsck hbck2 = doFsck(conf, false); + assertNoErrors(hbck2); + assertEquals(0, hbck2.getOverlapGroups(table).size()); + + List regions = cluster.getRegions(desc.getTableName()); + assertEquals(1, regions.size()); + // verify start-key and end-key of newly created region + assertTrue( + Arrays.equals(regions.get(0).getRegionInfo().getStartKey(), HConstants.EMPTY_START_ROW) + && Arrays.equals(regions.get(0).getRegionInfo().getEndKey(), HConstants.EMPTY_END_ROW)); + assertEquals(5, countRows()); + } finally { + cleanupTable(table); + } + } public static class MasterSyncObserver extends BaseMasterObserver { volatile CountDownLatch tableCreationLatch = null; -- 2.7.2.windows.1