diff --git src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java index 8873512..9c14612 100644 --- src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java +++ src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Writables; /** @@ -306,6 +307,16 @@ public class MetaEditor { return info; } + public static PairOfSameType getDaughterRegions(Result data) throws IOException { + HRegionInfo splitA = Writables.getHRegionInfoOrNull( + data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER)); + HRegionInfo splitB = Writables.getHRegionInfoOrNull( + data.getValue(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER)); + + return new PairOfSameType(splitA, splitB); + } + + private static Put addRegionInfo(final Put p, final HRegionInfo hri) throws IOException { p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index a8091e6..3237e32 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -17,18 +17,37 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.ArrayList; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Random; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.NotServingRegionException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.MetaScanner; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; @@ -37,15 +56,19 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.PairOfSameType; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; +import com.google.common.collect.Iterators; import com.google.protobuf.ServiceException; @Category(LargeTests.class) public class TestEndToEndSplitTransaction { + private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - + private static final Configuration conf = TEST_UTIL.getConfiguration(); + @BeforeClass public static void beforeAllTests() throws Exception { TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5); @@ -58,6 +81,7 @@ public class TestEndToEndSplitTransaction { } @Test + @Ignore public void testMasterOpsWhileSplitting() throws Exception { byte[] tableName = Bytes.toBytes("TestSplit"); byte[] familyName = Bytes.toBytes("fam"); @@ -138,6 +162,215 @@ public class TestEndToEndSplitTransaction { return true; } + /** + * Tests that the client sees meta table changes as atomic during splits + */ + @Test + public void testFromClientSideWhileSplitting() throws Throwable { + LOG.info("Starting testMetaScannerAtomicSplits"); + final byte[] TABLENAME = Bytes.toBytes("testAllTableRegionsWhileSplits"); + final byte[] FAMILY = Bytes.toBytes("family"); + + //SplitTransaction will update the meta table with offlining the parent region, and adding info + //for daughters. + HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY); + + //put some rows to the table + for (int i=0; i< 1000; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.add(FAMILY,FAMILY, Bytes.toBytes(i)); + table.put(put); + } + table.flushCommits(); + + Stoppable stopper = new SimpleStoppable(); + RegionSplitter regionSplitter = new RegionSplitter(TABLENAME); + RegionChecker regionChecker = new RegionChecker(stopper, TABLENAME); + + regionChecker.start(); + regionSplitter.start(); + + //wait until the splitter is finished + regionSplitter.join(); + stopper.stop(null); + + if (regionChecker.ex != null) { + throw regionChecker.ex; + } + + if (regionSplitter.ex != null) { + throw regionSplitter.ex; + } + + //one final check + regionChecker.verifyRegions(); + } + + static class SimpleStoppable implements Stoppable { + volatile boolean stopped = false; + + @Override + public void stop(String why) { + this.stopped = true; + } + + @Override + public boolean isStopped() { + return stopped; + } + } + + static class RegionSplitter extends Thread { + Throwable ex; + byte[] tableName; + HBaseAdmin admin; + HTable metaTable; + + RegionSplitter(byte[] tableName) throws IOException { + this.tableName = tableName; + admin = TEST_UTIL.getHBaseAdmin(); + metaTable = new HTable(conf, HConstants.META_TABLE_NAME); + } + + public void run() { + try { + Random random = new Random(); + ArrayList waitQueue = new ArrayList(); + for (int i=0; i< 3; i++) { + //pick a random region and split it into two + NavigableMap regions = MetaScanner.allTableRegions(conf, tableName, false); + if (regions.size() == 0) { + continue; + } + + int regionIndex = random.nextInt(regions.size()); + + HRegionInfo region = Iterators.get(regions.keySet().iterator(), regionIndex); + LOG.info("Initiating region split"); + try { + admin.split(region.getRegionName()); + //wait until the split is complete + blockUntilRegionSplit(5000, region.getRegionName(), false); + //don't wait for daughters to show up now, but add them for later + waitQueue.add(region.getRegionName()); + + } catch (NotServingRegionException ex) { + //ignore + } + } + + //while the SplitTransaction is run, we block indefinitely if the rs is shutting down + //so we wait until all splits are done + for (byte[] regionName : waitQueue) { + blockUntilRegionIsInMeta(3000, regionName); + } + + } catch (Throwable ex) { + this.ex = ex; + } finally { + if (metaTable != null) { + IOUtils.closeQuietly(metaTable); + } + } + } + + /** bloks until the meta shows that the region is split */ + void blockUntilRegionSplit(long timeout, final byte[] regionName, boolean waitForDaughters) + throws IOException, InterruptedException { + long start = System.currentTimeMillis(); + LOG.info("blocking until region is split:" + Bytes.toStringBinary(regionName)); + HRegionInfo daughterA = null, daughterB = null; + + while (System.currentTimeMillis() - start < timeout) { + Result result = getRegionRow(regionName); + if (result == null) { + break; + } + + HRegionInfo region = MetaEditor.getHRegionInfo(result); + if(region.isSplitParent()) { + PairOfSameType pair = MetaEditor.getDaughterRegions(result); + daughterA = pair.getFirst(); + daughterB = pair.getSecond(); + break; + } + sleep(10); + } + + //if we are here, this means the region split is complete or timed out + if (waitForDaughters) { + long rem = timeout - (System.currentTimeMillis() - start); + blockUntilRegionIsInMeta(rem, daughterA.getRegionName()); + + rem = timeout - (System.currentTimeMillis() - start); + blockUntilRegionIsInMeta(rem, daughterB.getRegionName()); + } + } + + Result getRegionRow(byte[] regionName) throws IOException { + Get get = new Get(regionName); + return metaTable.get(get); + } + + void blockUntilRegionIsInMeta(long timeout, byte[] regionName) + throws IOException, InterruptedException { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < timeout) { + Result result = getRegionRow(regionName); + if (result == null) { + continue; + } + sleep(10); + } + } + + } + + static class RegionChecker extends Chore { + Throwable ex; + byte[] tableName; + + RegionChecker(Stoppable stopper, byte[] tableName) { + super("RegionChecker", 10, stopper); + this.tableName = tableName; + this.setDaemon(true); + } + + void verifyRegions() throws Exception { + NavigableMap regions = MetaScanner.allTableRegions(conf, tableName, + false); + + LOG.info("Verifying " + regions.size() + " regions"); + assertTrue("Found 0 regions for the table", regions.size() > 0); + + byte[] prevEndKey = HConstants.EMPTY_START_ROW; + assertArrayEquals("Start key for the first region is not byte[0]", + HConstants.EMPTY_START_ROW, regions.firstKey().getStartKey()); + + // ensure that we do not have any gaps + for (Entry entry : regions.entrySet()) { + HRegionInfo region = entry.getKey(); + assertArrayEquals( + "Hole in .META. is detected. prevEndKey=" + Bytes.toStringBinary(prevEndKey) + + " ,regionStartKey=" + Bytes.toStringBinary(region.getStartKey()), prevEndKey, + region.getStartKey()); + prevEndKey = region.getEndKey(); + } + assertArrayEquals("End key for the last region is not byte[0]", HConstants.EMPTY_END_ROW, + regions.lastKey().getEndKey()); + } + + @Override + protected void chore() { + try { + verifyRegions(); + } catch (Throwable ex) { + this.ex = ex; + stopper.stop("caught exception"); + } + } + } + @org.junit.Rule public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu = new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();