diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index deae034..bef49f5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -1706,6 +1706,47 @@ public class Bytes { return binaryIncrementPos(val, amount); } + /** + * Increment the key to the next key + * @param key the key to increment + * @return a new byte array with the next key or null if the key could not be incremented because + * it's already at its max value. + */ + public static byte[] nextKey(byte[] key) { + byte[] nextStartRow = new byte[key.length]; + System.arraycopy(key, 0, nextStartRow, 0, key.length); + if (!nextKey(nextStartRow, nextStartRow.length)) { + return null; + } + return nextStartRow; + } + + /** + * Increment the key in-place to the next key + * @param key the key to increment + * @param length the length of the key + * @return true if the key can be incremented and false otherwise if the key is at its max value. + */ + public static boolean nextKey(byte[] key, int length) { + return nextKey(key, 0, length); + } + + public static boolean nextKey(byte[] key, int offset, int length) { + if (length == 0) { + return false; + } + int i = offset + length - 1; + while (key[i] == -1) { + key[i] = 0; + i--; + if (i < offset) { + return false; + } + } + key[i] = (byte) (key[i] + 1); + return true; + } + /* increment/deincrement for positive value */ private static byte [] binaryIncrementPos(byte [] value, long amount) { long amo = amount; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index d0c84b3..5655350 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -633,7 +633,7 @@ public class HBaseFsck extends Configured { FileSystem fs = p.getFileSystem(getConf()); FileStatus[] dirs = fs.listStatus(p); if (dirs == null) { - LOG.warn("Attempt to adopt ophan hdfs region skipped becuase no files present in " + + LOG.warn("Attempt to adopt orphan hdfs region skipped because no files present in " + p + ". This dir could probably be deleted."); return ; } @@ -674,6 +674,11 @@ public class HBaseFsck extends Configured { } } + // Start key and end key must not be same except when both are empty + // Otherwise, invalid region hri with same startkey and endkey will be created + if (!Bytes.equals(end, HConstants.EMPTY_END_ROW) && Bytes.equals(start, end)) { + end = Bytes.nextKey(start); + } // expand the range to include the range of all hfiles if (orphanRegionRange == null) { // first range @@ -2292,9 +2297,9 @@ public class HBaseFsck extends Configured { "Last region should end with an empty key. Creating a new " + "region and regioninfo in HDFS to plug the hole.", getTableInfo()); HTableDescriptor htd = getTableInfo().getHTD(); - // from curEndKey to EMPTY_START_ROW + // from curEndKey to EMPTY_END_ROW HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey, - HConstants.EMPTY_START_ROW); + HConstants.EMPTY_END_ROW); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("Table region end key was not empty. Created new empty region: " + newRegion diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 134a953..f41f817 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -126,6 +126,8 @@ public class TestHBaseFsck { private final static byte[][] ROWKEYS= new byte[][] { Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"), Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") }; + private final static byte[][] SINGLE_ROWKEY = new byte[][] { Bytes.toBytes("00") }; + private final static byte[][] EMPTY_ROWKEY = new byte[][] {}; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -349,6 +351,10 @@ public class TestHBaseFsck { dumpMeta(htd.getTableName()); } + // setupTable default with SPLITS + HTable setupTable(TableName tablename) throws Exception { + return setupTable(tablename, true, ROWKEYS); + } /** * Setup a clean table before we start mucking with it. * @@ -356,15 +362,19 @@ public class TestHBaseFsck { * @throws InterruptedException * @throws KeeperException */ - HTable setupTable(TableName tablename) throws Exception { + HTable setupTable(TableName tablename, boolean withSplits, byte[][] rowkeys) throws Exception { HTableDescriptor desc = new HTableDescriptor(tablename); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); desc.addFamily(hcd); // If a table has no CF's it doesn't get checked - TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS); + if (withSplits) { + TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS); + } else { + TEST_UTIL.getHBaseAdmin().createTable(desc); + } tbl = new HTable(TEST_UTIL.getConfiguration(), tablename, executorService); List puts = new ArrayList(); - for (byte[] row : ROWKEYS) { + for (byte[] row : rowkeys) { Put p = new Put(row); p.add(FAM, Bytes.toBytes("val"), row); puts.add(p); @@ -939,7 +949,7 @@ public class TestHBaseFsck { /** * This creates and fixes a bad table with a missing region -- hole in meta - * and data present but .regioinfino missing (an orphan hdfs region)in the fs. + * and data present but .regioninfo missing (an orphan hdfs region)in the fs. */ @Test public void testHDFSRegioninfoMissing() throws Exception { @@ -975,6 +985,132 @@ public class TestHBaseFsck { } /** + * This creates and fixes a bad table with single region with .regioninfo missing -- hole in meta + * and data present but .regioninfo missing (an orphan hdfs region)in the fs. + */ + @Test + public void testHDFSRegioninfoMissingInSingleRegion() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + try { + setupTable(table, false, ROWKEYS); + assertEquals(ROWKEYS.length, countRows()); + // Flush the data for HFiles to be available + TEST_UTIL.getHBaseAdmin().flush(table.getName()); + // Mess it up by leaving a hole in the meta data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes(""), false, + false, false, true); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.ORPHAN_HDFS_REGION }); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(ROWKEYS.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with single region with .regioninfo missing -- hole in meta + * and no data and .regioninfo missing (an orphan hdfs region)in the fs. + */ + @Test + public void testHDFSRegioninfoMissingSingleRegionNoData() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + try { + setupTable(table, false, EMPTY_ROWKEY); + assertEquals(EMPTY_ROWKEY.length, countRows()); + + // Mess it up by leaving a hole in the meta data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes(""), false, + false, false, true); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.ORPHAN_HDFS_REGION }); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(EMPTY_ROWKEY.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with splits/multiple regions with .regioninfo missing -- + * hole in meta and no data and .regioninfo missing (an orphan hdfs region)in the fs. + */ + @Test + public void testHDFSRegioninfoMissingMultipleRegionNoData() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + try { + setupTable(table, true, EMPTY_ROWKEY); + assertEquals(EMPTY_ROWKEY.length, countRows()); + + // Mess it up by leaving a hole in the meta data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"), Bytes.toBytes("C"), false, + false, false, true); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.ORPHAN_HDFS_REGION }); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(EMPTY_ROWKEY.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** + * This creates and fixes a bad table with single region and single keyvalue with .regioninfo missing + * -- hole in meta and data present but .regioninfo missing (an orphan hdfs region)in the fs. + */ + @Test + public void testHDFSRegioninfoMissingSingleRegionSingleKV() throws Exception { + TableName table = TableName.valueOf(name.getMethodName()); + try { + setupTable(table, false, SINGLE_ROWKEY); + assertEquals(SINGLE_ROWKEY.length, countRows()); + // Flush the data for HFiles to be available + TEST_UTIL.getHBaseAdmin().flush(table.getName()); + // Mess it up by leaving a hole in the meta data + deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes(""), false, + false, false, true); + + HBaseFsck hbck = doFsck(conf, false); + assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.ORPHAN_HDFS_REGION }); + // holes are separate from overlap groups + assertEquals(0, hbck.getOverlapGroups(table).size()); + + // fix hole + doFsck(conf, true); + + // check that hole fixed + assertNoErrors(doFsck(conf, false)); + assertEquals(SINGLE_ROWKEY.length, countRows()); + } finally { + deleteTable(table); + } + } + + /** * This creates and fixes a bad table with a region that is missing meta and * not assigned to a region server. */