Index: hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java =================================================================== --- hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (revision 1463118) +++ hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (working copy) @@ -346,16 +346,22 @@ /** * If the table is found in ENABLING state the inmemory state is removed. This * helps in cases where CreateTable is to be retried by the client incase of - * failures + * failures. If deleteZNode is true - the znode is also deleted * * @param tableName + * @param deleteZNode + * @throws KeeperException */ - public void removeEnablingTable(final String tableName) { + public void removeEnablingTable(final String tableName, boolean deleteZNode) + throws KeeperException { synchronized (this.cache) { if (isEnablingTable(tableName)) { this.cache.remove(tableName); + if (deleteZNode) { + ZKUtil.deleteNodeFailSilent(this.watcher, + ZKUtil.joinZNode(this.watcher.tableZNode, tableName)); + } } - } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (revision 1463118) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (working copy) @@ -172,8 +172,14 @@ // If this does not happen then if the client tries to create the table // again with the same Active master // It will block the creation saying TableAlreadyExists. - this.assignmentManager.getZKTable().removeEnablingTable( - this.hTableDescriptor.getNameAsString()); + try { + this.assignmentManager.getZKTable().removeEnablingTable( + this.hTableDescriptor.getNameAsString(), false); + } catch (KeeperException e) { + // Keeper exception should not happen here + LOG.error("Got a keeper exception while removing the ENABLING table znode " + + this.hTableDescriptor.getNameAsString(), e); + } } } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (revision 1463118) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (working copy) @@ -86,7 +86,17 @@ try { // Check if table exists if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) { - throw new TableNotFoundException(Bytes.toString(tableName)); + // retainAssignment is true only during recovery. In normal case it is false + if (!this.retainAssignment) { + throw new TableNotFoundException(tableNameStr); + } + try { + this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true); + } catch (KeeperException e) { + // TODO : Use HBCK to clear such nodes + LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr + + ". The table will remain unusable. Run HBCK to manually fix the problem."); + } } // There could be multiple client requests trying to disable or enable Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java (revision 1463118) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java (working copy) @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.concurrent.CountDownLatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -41,7 +42,9 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZKTable; import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -52,16 +55,18 @@ private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class); private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler"); private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); - public static boolean throwException = false; + private static boolean throwException = false; + - @BeforeClass - public static void setUp() throws Exception { + @Before + public void setUp() throws Exception { TEST_UTIL.startMiniCluster(1); } - @AfterClass - public static void tearDown() throws Exception { + @After + public void tearDown() throws Exception { TEST_UTIL.shutdownMiniCluster(); + throwException = false; } @Test @@ -90,6 +95,34 @@ assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME)); } + + @Test (timeout=5000) + public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception { + byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated"); + final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + final HMaster m = cluster.getMaster(); + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILYNAME)); + final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null, + null) }; + CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(), + desc, cluster.getConfiguration(), hRegionInfos, m); + handler.prepare(); + throwException = true; + handler.process(); + abortAndStartNewMaster(cluster); + assertTrue(cluster.getLiveMasterThreads().size() == 1); + + } + + private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException { + cluster.abortMaster(0); + cluster.waitOnMaster(0); + LOG.info("Starting new master"); + cluster.startMaster(); + LOG.info("Waiting for master to become active."); + cluster.waitForActiveAndReadyMaster(); + } private static class CustomCreateTableHandler extends CreateTableHandler { public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager,