Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java (revision 0) @@ -0,0 +1,192 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mortbay.log.Log; + +/** + * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction} + * only the below tests are against a running cluster where {@link TestSplitTransaction} + * is tests against a bare {@link HRegion}. + */ +public class TestSplitTransactionOnCluster { + private static final HBaseTestingUtility TESTING_UTIL = + new HBaseTestingUtility(); + + @BeforeClass public static void before() throws Exception { + TESTING_UTIL.startMiniCluster(2); + } + + @AfterClass public static void after() throws Exception { + TESTING_UTIL.shutdownMiniCluster(); + } + + /** + * Messy test that simulates case where SplitTransactions fails to add one + * of the daughters up into the .META. table before crash. We're testing + * fact that the shutdown handler will fixup the missing daughter region + * adding it back into .META. + * @throws IOException + * @throws InterruptedException + */ + @Test public void testShutdownMetaFixupHBASE3403() + throws IOException, InterruptedException { + final byte [] tableName = Bytes.toBytes("testShutdownMetaFixupHBASE3403"); + final byte [] columnFamily = HConstants.CATALOG_FAMILY; + HBaseAdmin admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); + // Print out what regions are where. Will help debugging if issue. + MiniHBaseCluster cluster = TESTING_UTIL.getMiniHBaseCluster(); + for (RegionServerThread rst: cluster.getRegionServerThreads()) { + Log.info("Server " + rst.getName() + ", regions=" + + rst.getRegionServer().getOnlineRegions()); + } + + // Create table then get the single region for our new table. + HTable t = TESTING_UTIL.createTable(tableName, columnFamily); + List regions = cluster.getRegions(tableName); + assertEquals(1, regions.size()); + HRegionInfo hri = regions.get(0).getRegionInfo(); + + int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); + + // Turn off balancer so it doesn't cut in and mess up our placements. + admin.balanceSwitch(false); + // Turn off the meta scanner so it don't remove parent on us. + cluster.getMaster().catalogJanitorSwitch(false); + try { + // Add a bit of load up into the table so splittable. + TESTING_UTIL.loadTable(t, columnFamily); + // Get region pre-split. + HRegionServer server = cluster.getRegionServer(tableRegionIndex); + printOutRegions(server, "Initial regions: "); + int regionCount = server.getOnlineRegions().size(); + // Now split. + admin.split(hri.getRegionNameAsString()); + while(server.getOnlineRegions().size() <= regionCount) { + Log.debug("Waiting on region to split"); + Thread.sleep(100); + } + // Get daughters + List daughters = cluster.getRegions(tableName); + assertTrue(daughters.size() >= 2); + // Remove one of the daughters from .META. to simulate failed insert of + // daughter region up into .META. + removeDaughterFromMeta(daughters.get(0).getRegionName()); + // Now crash the server + cluster.abortRegionServer(tableRegionIndex); + while(server.getOnlineRegions().size() > 0) { + Log.info("Waiting on server to go down"); + Thread.sleep(100); + } + // Wait till regions are back on line again. + while(cluster.getRegions(tableName).size() < 2) { + Log.info("Waiting for repair to happen"); + Thread.sleep(1000); + } + regions = cluster.getRegions(tableName); + for (HRegion r: regions) { + assertTrue(daughters.contains(r)); + } + } finally { + admin.balanceSwitch(true); + cluster.getMaster().catalogJanitorSwitch(true); + } + } + + private void removeDaughterFromMeta(final byte [] regionName) throws IOException { + HTable metaTable = + new HTable(TESTING_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); + Delete d = new Delete(regionName); + metaTable.delete(d); + } + + /** + * Ensure single table region is not on same server as the single .META. table + * region. + * @param admin + * @param hri + * @return Index of the server hosting the single table region + * @throws UnknownRegionException + * @throws MasterNotRunningException + * @throws ZooKeeperConnectionException + * @throws InterruptedException + */ + private int ensureTableRegionNotOnSameServerAsMeta(final HBaseAdmin admin, + final HRegionInfo hri) + throws UnknownRegionException, MasterNotRunningException, + ZooKeeperConnectionException, InterruptedException { + MiniHBaseCluster cluster = TESTING_UTIL.getMiniHBaseCluster(); + // Now make sure that the table region is not on same server as that hosting + // .META. We don't want .META. replay polluting our test when we later crash + // the table region serving server. + int metaServerIndex = cluster.getServerWithMeta(); + assertTrue(metaServerIndex != -1); + int tableRegionIndex = cluster.getServerWith(hri.getRegionName()); + assertTrue(tableRegionIndex != -1); + if (metaServerIndex == tableRegionIndex) { + // Server indexes are zero-based. If not 0, then server will be '1'. + int index = metaServerIndex == 0? 1: 0; + admin.move(hri.getEncodedNameAsBytes(), + Bytes.toBytes(cluster.getRegionServer(index).getServerName())); + } + // Wait till table region is up on the server that is NOT carrying .META.. + while (true) { + tableRegionIndex = cluster.getServerWith(hri.getRegionName()); + if (tableRegionIndex != -1 && tableRegionIndex != metaServerIndex) break; + Log.debug("Waiting on region move off the .META. server; current index " + + tableRegionIndex); + Thread.sleep(100); + } + // Verify for sure table region is not on same server as .META. + tableRegionIndex = cluster.getServerWith(hri.getRegionName()); + assertTrue(tableRegionIndex != -1); + assertNotSame(metaServerIndex, tableRegionIndex); + return tableRegionIndex; + } + + private void printOutRegions(final HRegionServer hrs, final String prefix) { + List regions = hrs.getOnlineRegions(); + for (HRegionInfo region: regions) { + Log.info(prefix + region.getRegionNameAsString()); + } + } +} \ No newline at end of file Index: src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java (revision 1054820) +++ src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java (working copy) @@ -51,6 +51,7 @@ private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName()); private final Server server; private final MasterServices services; + private boolean onOff = true; CatalogJanitor(final Server server, final MasterServices services) { super(server.getServerName() + "-CatalogJanitor", @@ -63,7 +64,7 @@ @Override protected boolean initialChore() { try { - scan(); + if (this.onOff) scan(); } catch (IOException e) { LOG.warn("Failed initial scan of catalog table", e); return false; @@ -71,6 +72,11 @@ return true; } + public boolean onOff(final boolean onOff) { + this.onOff = onOff; + return this.onOff; + } + @Override protected void chore() { try { Index: src/main/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 1054820) +++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -741,6 +741,16 @@ return oldValue; } + /** + * Switch for the background {@link CatalogJanitor} thread. + * Used testing. + * @param b If false, the catalog janitor won't run. + * @return New state of the catalog janitor flag. + */ + public boolean catalogJanitorSwitch(final boolean b) { + return ((CatalogJanitor)this.catalogJanitorChore).onOff(b); + } + @Override public void move(final byte[] encodedRegionName, final byte[] destServerName) throws UnknownRegionException { Index: src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (revision 1054820) +++ src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (working copy) @@ -77,10 +77,6 @@ copyOfParent.setSplit(true); Put put = new Put(copyOfParent.getRegionName()); addRegionInfo(put, copyOfParent); - put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, - HConstants.EMPTY_BYTE_ARRAY); - put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, - HConstants.EMPTY_BYTE_ARRAY); put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER, Writables.getBytes(a)); put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,