Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1449369) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; @@ -68,6 +69,7 @@ import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.KeyLocker; import org.apache.hadoop.hbase.util.Pair; @@ -84,6 +86,7 @@ import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; +import com.google.common.base.Preconditions; import com.google.common.collect.LinkedHashMultimap; /** @@ -109,6 +112,8 @@ final TimeoutMonitor timeoutMonitor; private TimerUpdater timerUpdater; + + private TableStateUpdater tableStateUpdater; private LoadBalancer balancer; @@ -210,6 +215,10 @@ conf.getInt("hbase.master.assignment.timeoutmonitor.timeout", 600000)); this.timerUpdater = new TimerUpdater(conf.getInt( "hbase.master.assignment.timerupdater.period", 10000), server); + this.tableStateUpdater = new TableStateUpdater(conf.getInt( + "hbase.master.assignment.tablestateupdater.period", 120000), server); + Threads.setDaemonThreadRunning(tableStateUpdater.getThread(), server.getServerName() + + ".tableStateUpdater"); Threads.setDaemonThreadRunning(timerUpdater.getThread(), server.getServerName() + ".timerUpdater"); this.zkTable = new ZKTable(this.watcher); @@ -2687,6 +2696,61 @@ } } + /** + * Enable/disable the tables again if something wrong happened in the middle. + */ + protected class TableStateUpdater extends Chore { + + Map tablesInPartialState = + new TreeMap(Bytes.BYTES_COMPARATOR); + + public TableStateUpdater(final int period, final Stoppable stopper) { + super("TableStateUpdater", period, stopper); + } + + @Override + protected void chore() { + synchronized (this.tablesInPartialState) { + for (Iterator> tableItr = + tablesInPartialState.entrySet().iterator(); tableItr.hasNext();) { + Entry tableEntry = tableItr.next(); + int onlineRegionsCount = regionStates.getRegionsOfTable(tableEntry.getKey()).size(); + if (tableEntry.getValue() != 0) { + Preconditions.checkArgument(onlineRegionsCount <= tableEntry.getValue(), + "Number of online regions should not be more than total regions count."); + } + if (tableEntry.getValue() == onlineRegionsCount) { + boolean enabled = false; + try { + if (tableEntry.getValue() != 0) { // total regions got assigned. + enabled = true; + zkTable.setEnabledTable(Bytes.toString(tableEntry.getKey())); + } else { // zero means all the regions unassigned. + zkTable.setDisabledTable(Bytes.toString(tableEntry.getKey())); + } + tableItr.remove(); + } catch (KeeperException e) { + LOG.warn("Error trying to set " + (enabled ? "enabled" : "disabled") + + " state for table " + Bytes.toString(tableEntry.getKey()) + " in zookeeper.", + e); + } + } + } + } + } + + private void addTableInPartialState(byte[] tableName, int expectedOnlineRegionsCount) { + synchronized (this.tablesInPartialState) { + this.tablesInPartialState.put(tableName, expectedOnlineRegionsCount); + } + } + + } + + public void addTableInPartialState(byte[] tableName, int expectedOnlineRegionsCount) { + this.tableStateUpdater.addTableInPartialState(tableName, expectedOnlineRegionsCount); + } + private void processOpeningState(HRegionInfo regionInfo) { LOG.info("Region has been OPENING for too long, reassigning region=" + regionInfo.getRegionNameAsString()); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java (revision 1449369) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DisableTableHandler.java (working copy) @@ -182,7 +182,11 @@ } } // Flip the table to disabled if success. - if (done) this.assignmentManager.getZKTable().setDisabledTable(this.tableNameStr); + if (done){ + this.assignmentManager.getZKTable().setDisabledTable(this.tableNameStr); + } else { + this.assignmentManager.addTableInPartialState(this.tableName, 0); + } LOG.info("Disabled table is done=" + done); } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (revision 1449369) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (working copy) @@ -195,6 +195,7 @@ LOG.info("Table '" + this.tableNameStr + "' was successfully enabled. Status: done=" + done); } else { + this.assignmentManager.addTableInPartialState(this.tableName, countOfRegionsInTable); LOG.warn("Table '" + this.tableNameStr + "' wasn't successfully enabled. Status: done=" + done); } Index: hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java (revision 1449369) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java (working copy) @@ -32,10 +32,17 @@ import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.master.handler.EnableTableHandler; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZKTable; +import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -53,6 +60,7 @@ @BeforeClass public static void setUpBeforeClass() throws Exception { + conf.setInt("hbase.master.assignment.tablestateupdater.period", 1000); TEST_UTIL.startMiniCluster(3); admin = TEST_UTIL.getHBaseAdmin(); } @@ -168,11 +176,52 @@ } } + @Test + public void testTableStateUpdaterInCaseOfEnableOrDisableOfATableFailed() throws Exception { + String table = "testTableStateUpdaterInCaseOfEnableOrDisableOfATableFailed"; + HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + AssignmentManager am = master.getAssignmentManager(); + Configuration conf = master.getConfiguration(); + ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, table, null); + try { + createTable(table); + am.getZKTable().setDisablingTable(table); + am.addTableInPartialState(Bytes.toBytes(table), 0); + List> tableRegionsAndLocations = MetaReader + .getTableRegionsAndLocations(master.getCatalogTracker(), Bytes.toBytes(table), true); + for (Pair pair : tableRegionsAndLocations) { + am.regionOffline(pair.getFirst()); + } + long startTime = System.currentTimeMillis(); + long timeout = 5000; + long remaining = timeout; + while (!ZKTableReadOnly.isDisabledTable(zkw, table) && remaining >= 0) { + Thread.sleep(1000); + remaining = timeout - (System.currentTimeMillis() - startTime); + } + assertTrue("Table should be in disabled state.", + ZKTableReadOnly.isDisabledTable(zkw, table)); + am.getZKTable().setEnablingTable(table); + am.addTableInPartialState(Bytes.toBytes(table), 5); + for (Pair pair : tableRegionsAndLocations) { + am.regionOnline(pair.getFirst(), pair.getSecond()); + } + startTime = System.currentTimeMillis(); + remaining = timeout; + while (!ZKTableReadOnly.isEnabledTable(zkw, table) && remaining >= 0) { + Thread.sleep(1000); + remaining = timeout - (System.currentTimeMillis() - startTime); + } + assertTrue("Table should be in enabled state.", + ZKTableReadOnly.isEnabledTable(zkw, table)); + } finally { + TEST_UTIL.deleteTable(Bytes.toBytes(table)); + } + } + HRegionInfo createTableAndGetOneRegion( final String tableName) throws IOException, InterruptedException { - HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor(FAMILY)); - admin.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), 5); + createTable(tableName); // wait till the table is assigned HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); @@ -191,4 +240,10 @@ } } + private void createTable(final String tableName) throws IOException { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + admin.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), 5); + } + }