diff --git src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java index ae31fb0..482ab0b 100644 --- src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java +++ src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; +import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; /** * Handles everything on master-side related to master election. @@ -118,7 +119,8 @@ class ActiveMasterManager extends ZooKeeperListener { * master was running or if some other problem (zookeeper, stop flag has been * set on this Master) */ - boolean blockUntilBecomingActiveMaster() { + boolean blockUntilBecomingActiveMaster( + ClusterStatusTracker clusterStatusTracker) { boolean cleanSetOfActiveMaster = true; // Try to become the active master, watch if there is another master try { @@ -158,11 +160,14 @@ class ActiveMasterManager extends ZooKeeperListener { LOG.debug("Interrupted waiting for master to die", e); } } + if (!clusterStatusTracker.isClusterUp()) { + this.master.stop("Cluster went down before this master became active"); + } if (this.master.isStopped()) { return cleanSetOfActiveMaster; } // Try to become active master again now that there is no active master - blockUntilBecomingActiveMaster(); + blockUntilBecomingActiveMaster(clusterStatusTracker); } return cleanSetOfActiveMaster; } diff --git src/main/java/org/apache/hadoop/hbase/master/HMaster.java src/main/java/org/apache/hadoop/hbase/master/HMaster.java index b66722a..5765fef 100644 --- src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -275,7 +275,15 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { this.activeMasterManager = new ActiveMasterManager(zooKeeper, address, this); this.zooKeeper.registerListener(activeMasterManager); stallIfBackupMaster(this.conf, this.activeMasterManager); - this.activeMasterManager.blockUntilBecomingActiveMaster(); + + // The ClusterStatusTracker is setup before the other + // ZKBasedSystemTrackers because it's needed by the activeMasterManager + // to check if the cluster should be shutdown. + this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this); + this.clusterStatusTracker.start(); + this.activeMasterManager.blockUntilBecomingActiveMaster( + this.clusterStatusTracker); + // We are either the active master or we were asked to shutdown if (!this.stopped) { finishInitialization(); @@ -361,8 +369,6 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { // Set the cluster as up. If new RSs, they'll be waiting on this before // going ahead with their startup. - this.clusterStatusTracker = new ClusterStatusTracker(getZooKeeper(), this); - this.clusterStatusTracker.start(); boolean wasUp = this.clusterStatusTracker.isClusterUp(); if (!wasUp) this.clusterStatusTracker.setClusterUp(); diff --git src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java index 1a19941..34f7d40 100644 --- src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java +++ src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -64,29 +65,32 @@ public class TestActiveMasterManager { ZKUtil.createAndFailSilent(zk, zk.baseZNode); try { ZKUtil.deleteNode(zk, zk.masterAddressZNode); + ZKUtil.deleteNode(zk, zk.clusterStateZNode); } catch(KeeperException.NoNodeException nne) {} // Create the master node with a dummy address HServerAddress master = new HServerAddress("localhost", 1); // Should not have a master yet - DummyMaster dummyMaster = new DummyMaster(); - ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk, - master, dummyMaster); - zk.registerListener(activeMasterManager); + DummyMaster dummyMaster = new DummyMaster(zk,master); + ClusterStatusTracker clusterStatusTracker = + dummyMaster.getClusterStatusTracker(); + ActiveMasterManager activeMasterManager = + dummyMaster.getActiveMasterManager(); assertFalse(activeMasterManager.clusterHasActiveMaster.get()); // First test becoming the active master uninterrupted - activeMasterManager.blockUntilBecomingActiveMaster(); + clusterStatusTracker.setClusterUp(); + + activeMasterManager.blockUntilBecomingActiveMaster(clusterStatusTracker); assertTrue(activeMasterManager.clusterHasActiveMaster.get()); assertMaster(zk, master); // Now pretend master restart - DummyMaster secondDummyMaster = new DummyMaster(); - ActiveMasterManager secondActiveMasterManager = new ActiveMasterManager(zk, - master, secondDummyMaster); - zk.registerListener(secondActiveMasterManager); + DummyMaster secondDummyMaster = new DummyMaster(zk,master); + ActiveMasterManager secondActiveMasterManager = + secondDummyMaster.getActiveMasterManager(); assertFalse(secondActiveMasterManager.clusterHasActiveMaster.get()); - activeMasterManager.blockUntilBecomingActiveMaster(); + activeMasterManager.blockUntilBecomingActiveMaster(clusterStatusTracker); assertTrue(activeMasterManager.clusterHasActiveMaster.get()); assertMaster(zk, master); } @@ -103,6 +107,7 @@ public class TestActiveMasterManager { ZKUtil.createAndFailSilent(zk, zk.baseZNode); try { ZKUtil.deleteNode(zk, zk.masterAddressZNode); + ZKUtil.deleteNode(zk, zk.clusterStateZNode); } catch(KeeperException.NoNodeException nne) {} // Create the master node with a dummy address @@ -110,20 +115,21 @@ public class TestActiveMasterManager { HServerAddress secondMasterAddress = new HServerAddress("localhost", 2); // Should not have a master yet - DummyMaster ms1 = new DummyMaster(); - ActiveMasterManager activeMasterManager = new ActiveMasterManager(zk, - firstMasterAddress, ms1); - zk.registerListener(activeMasterManager); + DummyMaster ms1 = new DummyMaster(zk,firstMasterAddress); + ActiveMasterManager activeMasterManager = + ms1.getActiveMasterManager(); assertFalse(activeMasterManager.clusterHasActiveMaster.get()); // First test becoming the active master uninterrupted - activeMasterManager.blockUntilBecomingActiveMaster(); + ClusterStatusTracker clusterStatusTracker = + ms1.getClusterStatusTracker(); + clusterStatusTracker.setClusterUp(); + activeMasterManager.blockUntilBecomingActiveMaster(clusterStatusTracker); assertTrue(activeMasterManager.clusterHasActiveMaster.get()); assertMaster(zk, firstMasterAddress); // New manager will now try to become the active master in another thread WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress); - zk.registerListener(t.manager); t.start(); // Wait for this guy to figure out there is another active master // Wait for 1 second at most @@ -186,18 +192,20 @@ public class TestActiveMasterManager { public static class WaitToBeMasterThread extends Thread { ActiveMasterManager manager; + DummyMaster dummyMaster; boolean isActiveMaster; public WaitToBeMasterThread(ZooKeeperWatcher zk, HServerAddress address) { - this.manager = new ActiveMasterManager(zk, address, - new DummyMaster()); + this.dummyMaster = new DummyMaster(zk,address); + this.manager = this.dummyMaster.getActiveMasterManager(); isActiveMaster = false; } @Override public void run() { - manager.blockUntilBecomingActiveMaster(); + manager.blockUntilBecomingActiveMaster( + this.dummyMaster.getClusterStatusTracker()); LOG.info("Second master has become the active master!"); isActiveMaster = true; } @@ -233,6 +241,18 @@ public class TestActiveMasterManager { */ public static class DummyMaster implements Server { private volatile boolean stopped; + private ClusterStatusTracker clusterStatusTracker; + private ActiveMasterManager activeMasterManager; + + public DummyMaster(ZooKeeperWatcher zk, HServerAddress master) { + this.clusterStatusTracker = + new ClusterStatusTracker(zk, this); + clusterStatusTracker.start(); + + this.activeMasterManager = + new ActiveMasterManager(zk, master, this); + zk.registerListener(activeMasterManager); + } @Override public void abort(final String msg, final Throwable t) {} @@ -266,5 +286,13 @@ public class TestActiveMasterManager { public CatalogTracker getCatalogTracker() { return null; } + + public ClusterStatusTracker getClusterStatusTracker() { + return clusterStatusTracker; + } + + public ActiveMasterManager getActiveMasterManager() { + return activeMasterManager; + } } } diff --git src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java new file mode 100644 index 0000000..ac8b2ad --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/master/TestMasterShutdown.java @@ -0,0 +1,90 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.hbase.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +public class TestMasterShutdown { + private static final Log LOG = LogFactory.getLog(TestMasterShutdown.class); + + /** + * Simple test of shutdown. + *

+ * Starts with three masters. Tells the active master to shutdown the cluster. + * Verifies that all masters are properly shutdown. + * @throws Exception + */ + @Test (timeout=240000) + public void testMasterShutdown() throws Exception { + + final int NUM_MASTERS = 3; + final int NUM_RS = 3; + + // Create config to use for this cluster + Configuration conf = HBaseConfiguration.create(); + + // Start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + + // get all the master threads + List masterThreads = cluster.getMasterThreads(); + + // wait for each to come online + for (MasterThread mt : masterThreads) { + assertTrue(mt.isAlive()); + } + + // find the active master + HMaster active = null; + for (int i = 0; i < masterThreads.size(); i++) { + if (masterThreads.get(i).getMaster().isActiveMaster()) { + active = masterThreads.get(i).getMaster(); + break; + } + } + assertNotNull(active); + + // tell the active master to shutdown the cluster + active.shutdown(); + + for (int i = NUM_MASTERS - 1; i >= 0 ;--i) { + cluster.waitOnMaster(i); + } + // make sure all the masters properly shutdown + assertEquals(0,masterThreads.size()); + + TEST_UTIL.shutdownMiniCluster(); + } +} +