Index: src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (revision 1023135) +++ src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (working copy) @@ -85,6 +85,11 @@ private final static Log LOG = LogFactory.getLog(HBaseTestingUtility.class); private Configuration conf; private MiniZooKeeperCluster zkCluster = null; + /** + * Set if we were passed a zkCluster. If so, we won't shutdown zk as + * part of general shutdown. + */ + private boolean passedZkCluster = false; private MiniDFSCluster dfsCluster = null; private MiniHBaseCluster hbaseCluster = null; private MiniMRCluster mrCluster = null; @@ -248,6 +253,7 @@ private MiniZooKeeperCluster startMiniZKCluster(final File dir) throws Exception { + this.passedZkCluster = false; if (this.zkCluster != null) { throw new IOException("Cluster already running at " + dir); } @@ -408,7 +414,7 @@ // Wait till hbase is down before going on to shutdown zk. this.hbaseCluster.join(); } - shutdownMiniZKCluster(); + if (!this.passedZkCluster) shutdownMiniZKCluster(); if (this.dfsCluster != null) { // The below throws an exception per dn, AsynchronousCloseException. this.dfsCluster.shutdown(); @@ -985,6 +991,7 @@ } public void setZkCluster(MiniZooKeeperCluster zkCluster) { + this.passedZkCluster = true; this.zkCluster = zkCluster; } Index: src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java (revision 1023135) +++ src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java (working copy) @@ -20,7 +20,9 @@ package org.apache.hadoop.hbase; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -29,6 +31,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; import org.junit.AfterClass; @@ -62,6 +69,65 @@ public void tearDown() throws Exception { } + /** + * Basic sanity test that spins up multiple HDFS and HBase clusters that share + * the same ZK ensemble. We then create the same table in both and make sure + * that what we insert in one place doesn't end up in the other. + * @throws Exception + */ + @Test (timeout=180000) + public void multiClusters() throws Exception { + // Create three clusters + + // Cluster 1. + HBaseTestingUtility htu1 = new HBaseTestingUtility(); + // Set a different zk path for each cluster + htu1.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + htu1.startMiniZKCluster(); + + // Cluster 2 + HBaseTestingUtility htu2 = new HBaseTestingUtility(); + htu2.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); + htu2.setZkCluster(htu1.getZkCluster()); + + // Cluster 3. + HBaseTestingUtility htu3 = new HBaseTestingUtility(); + htu3.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3"); + htu3.setZkCluster(htu1.getZkCluster()); + + try { + htu1.startMiniCluster(); + htu2.startMiniCluster(); + htu3.startMiniCluster(); + + final byte[] TABLE_NAME = Bytes.toBytes("test"); + final byte[] FAM_NAME = Bytes.toBytes("fam"); + final byte[] ROW = Bytes.toBytes("row"); + final byte[] QUAL_NAME = Bytes.toBytes("qual"); + final byte[] VALUE = Bytes.toBytes("value"); + + HTable table1 = htu1.createTable(TABLE_NAME, FAM_NAME); + HTable table2 = htu2.createTable(TABLE_NAME, FAM_NAME); + + Put put = new Put(ROW); + put.add(FAM_NAME, QUAL_NAME, VALUE); + table1.put(put); + + Get get = new Get(ROW); + get.addColumn(FAM_NAME, QUAL_NAME); + Result res = table1.get(get); + assertEquals(1, res.size()); + + res = table2.get(get); + assertEquals(0, res.size()); + + } finally { + htu3.shutdownMiniCluster(); + htu2.shutdownMiniCluster(); + htu1.shutdownMiniCluster(); + } + } + @Test public void testMiniCluster() throws Exception { MiniHBaseCluster cluster = this.hbt.startMiniCluster(); try { Index: src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java (revision 1023135) +++ src/test/java/org/apache/hadoop/hbase/replication/TestReplication.java (working copy) @@ -47,7 +47,6 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; public class TestReplication { @@ -170,10 +169,8 @@ */ @AfterClass public static void tearDownAfterClass() throws Exception { - /* REENABLE utility2.shutdownMiniCluster(); utility1.shutdownMiniCluster(); - */ } /** Index: src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java (revision 1023135) +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java (working copy) @@ -30,6 +30,8 @@ import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.ZooKeeperConnectionException; +import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.util.Threads; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; @@ -86,6 +88,10 @@ // znode used for table disabling/enabling public String tableZNode; + private final Configuration conf; + + private final Exception constructorCaller; + /** * Instantiate a ZooKeeper connection and watcher. * @param descriptor Descriptive string that is added to zookeeper sessionid @@ -96,6 +102,14 @@ public ZooKeeperWatcher(Configuration conf, String descriptor, Abortable abortable) throws IOException, ZooKeeperConnectionException { + this.conf = conf; + // Capture a stack trace now. Will print it out later if problem so we can + // distingush amongst the myriad ZKWs. + try { + throw new Exception("ZKW CONSTRUCTOR STACK TRACE FOR DEBUGGING"); + } catch (Exception e) { + this.constructorCaller = e; + } this.quorum = ZKConfig.getZKQuorumServersString(conf); // Identifier will get the sessionid appended later below down when we // handle the syncconnect event. @@ -270,6 +284,20 @@ case SyncConnected: // Update our identifier. Otherwise ignore. LOG.info(this.identifier + " connected"); + // Now, this callback can be invoked before the this.zookeeper is set. + // Wait a little while. + long finished = System.currentTimeMillis() + + this.conf.getLong("hbase.zookeeper.watcher.sync.connected.wait", 2000); + while (System.currentTimeMillis() < finished) { + Threads.sleep(1); + if (this.zooKeeper != null) break; + } + if (this.zooKeeper == null) { + LOG.error("ZK is null on connection event -- see stack trace " + + "for the stack trace when constructor was called on this zkw", + this.constructorCaller); + throw new NullPointerException("ZK is null"); + } this.identifier = this.identifier + "-0x" + Long.toHexString(this.zooKeeper.getSessionId()); break; Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1023135) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -699,7 +699,7 @@ this.regionPlans.remove(state.getRegion().getEncodedName()); // Put in place a new plan and reassign. Calling getRegionPlan will add // a plan if none exists (We removed it in line above). - if (getRegionPlan(state) == null) return; + if (getRegionPlan(state, plan.getDestination()) == null) return; assign(state); } }