Index: src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (revision 1465321) +++ src/main/java/org/apache/hadoop/hbase/master/handler/CreateTableHandler.java (working copy) @@ -142,8 +142,14 @@ // again with the same Active master // It will block the creation saying TableAlreadyExists. if (exception != null) { - this.assignmentManager.getZKTable().removeEnablingTable( - this.hTableDescriptor.getNameAsString()); + try { + this.assignmentManager.getZKTable().removeEnablingTable( + this.hTableDescriptor.getNameAsString(), false); + } catch (KeeperException e) { + // Keeper exception should not happen here + LOG.error("Got a keeper exception while removing the ENABLING table znode " + + this.hTableDescriptor.getNameAsString(), e); + } } } Index: src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (revision 1465321) +++ src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (working copy) @@ -66,7 +66,18 @@ this.retainAssignment = skipTableStateCheck; // Check if table exists if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) { - throw new TableNotFoundException(Bytes.toString(tableName)); + // retainAssignment is true only during recovery. In normal case it is + // false + if (!this.retainAssignment) { + throw new TableNotFoundException(tableNameStr); + } + try { + this.assignmentManager.getZKTable().removeEnablingTable(tableNameStr, true); + } catch (KeeperException e) { + // TODO : Use HBCK to clear such nodes + LOG.warn("Failed to delete the ENABLING node for the table " + tableNameStr + + ". The table will remain unusable. Run HBCK to manually fix the problem."); + } } // There could be multiple client requests trying to disable or enable Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (revision 1465321) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKTable.java (working copy) @@ -180,13 +180,21 @@ /** * If the table is found in ENABLING state the inmemory state is removed. - * This helps in cases where CreateTable is to be retried by the client incase of failures + * This helps in cases where CreateTable is to be retried by the client incase of failures. + * If deleteZNode is true - the znode is also deleted * @param tableName + * @param deleteZNode + * @throws KeeperException */ - public void removeEnablingTable(final String tableName) { + public void removeEnablingTable(final String tableName, boolean deleteZNode) + throws KeeperException { synchronized (this.cache) { if (isEnablingTable(tableName)) { this.cache.remove(tableName); + if (deleteZNode) { + ZKUtil.deleteNodeFailSilent(this.watcher, + ZKUtil.joinZNode(this.watcher.masterTableZNode, tableName)); + } } } Index: src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java (revision 1465321) +++ src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java (working copy) @@ -46,7 +46,9 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZKTable; import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -57,16 +59,17 @@ private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class); private static final byte[] TABLENAME = Bytes.toBytes("TestCreateTableHandler"); private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); - public static boolean throwException = false; + private static boolean throwException = false; - @BeforeClass - public static void setUp() throws Exception { + @Before + public void setUp() throws Exception { TEST_UTIL.startMiniCluster(1); } - @AfterClass - public static void tearDown() throws Exception { + @After + public void tearDown() throws Exception { TEST_UTIL.shutdownMiniCluster(); + throwException = true; } @Test @@ -95,6 +98,34 @@ assertTrue(TEST_UTIL.getHBaseAdmin().isTableEnabled(TABLENAME)); } + + @Test (timeout=10000) + public void testMasterRestartAfterEnablingNodeIsCreated() throws Exception { + byte[] tableName = Bytes.toBytes("testMasterRestartAfterEnablingNodeIsCreated"); + final MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + final HMaster m = cluster.getMaster(); + final HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILYNAME)); + final HRegionInfo[] hRegionInfos = new HRegionInfo[] { new HRegionInfo(desc.getName(), null, + null) }; + CustomCreateTableHandler handler = new CustomCreateTableHandler(m, m.getMasterFileSystem(), + m.getServerManager(), desc, cluster.getConfiguration(), hRegionInfos, + m.getCatalogTracker(), m.getAssignmentManager()); + throwException = true; + handler.process(); + abortAndStartNewMaster(cluster); + assertTrue(cluster.getLiveMasterThreads().size() == 1); + + } + + private void abortAndStartNewMaster(final MiniHBaseCluster cluster) throws IOException { + cluster.abortMaster(0); + cluster.waitOnMaster(0); + LOG.info("Starting new master"); + cluster.startMaster(); + LOG.info("Waiting for master to become active."); + cluster.waitForActiveAndReadyMaster(); + } private static class CustomCreateTableHandler extends CreateTableHandler { public CustomCreateTableHandler(Server server, MasterFileSystem fileSystemManager,