From 1c615e96da9ad369a9082c2602a9ce8470eb1912 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Mon, 22 Jan 2018 15:06:04 +0800 Subject: [PATCH] HBASE-19838 A UT to expose the problem stably --- .../org/apache/hadoop/hbase/master/HMaster.java | 7 +- .../hbase/master/TestShutdownBackupMaster.java | 103 +++++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 09b18bc..d96b366 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -2605,7 +2605,7 @@ public class HMaster extends HRegionServer implements MasterServices { } @Override - public void abort(final String msg, final Throwable t) { + public void abort(String reason, Throwable cause) { if (isAborted() || isStopped()) { return; } @@ -2614,8 +2614,9 @@ public class HMaster extends HRegionServer implements MasterServices { LOG.error(HBaseMarkers.FATAL, "Master server abort: loaded coprocessors are: " + getLoadedCoprocessors()); } - if (t != null) { - LOG.error(HBaseMarkers.FATAL, msg, t); + String msg = "***** ABORTING master " + this + ": " + reason + " *****"; + if (cause != null) { + LOG.error(HBaseMarkers.FATAL, msg, cause); } else { LOG.error(HBaseMarkers.FATAL, msg); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java new file mode 100644 index 0000000..00c56fe --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestShutdownBackupMaster.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test to confirm that we will not hang when stop a backup master which is trying to become the + * active master. + */ +@Category({ MasterTests.class, MediumTests.class }) +public class TestShutdownBackupMaster { + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static volatile CountDownLatch ARRIVE; + + private static volatile CountDownLatch CONTINUE; + + public static final class MockHMaster extends HMaster { + + public MockHMaster(Configuration conf) throws IOException, KeeperException { + super(conf); + } + + @Override + void initClusterSchemaService() throws IOException, InterruptedException { + if (ARRIVE != null) { + ARRIVE.countDown(); + CONTINUE.await(); + } + super.initClusterSchemaService(); + } + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, MockHMaster.class, HMaster.class); + UTIL.startMiniCluster(2, 2); + UTIL.waitUntilAllSystemRegionsAssigned(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + // make sure that we can stop the cluster cleanly + UTIL.shutdownMiniCluster(); + } + + @Test + public void testShutdownWhileBecomingActive() throws InterruptedException { + MiniHBaseCluster cluster = UTIL.getHBaseCluster(); + HMaster activeMaster = null; + HMaster backupMaster = null; + for (MasterThread t : cluster.getMasterThreads()) { + if (t.getMaster().isActiveMaster()) { + activeMaster = t.getMaster(); + } else { + backupMaster = t.getMaster(); + } + } + assertNotNull(activeMaster); + assertNotNull(backupMaster); + ARRIVE = new CountDownLatch(1); + CONTINUE = new CountDownLatch(1); + activeMaster.abort("Aborting active master for test"); + // wait until we arrive the initClusterSchemaService + ARRIVE.await(); + // killall RSes + cluster.getRegionServerThreads().stream().map(t -> t.getRegionServer()) + .forEachOrdered(rs -> rs.abort("Aborting RS for test")); + CONTINUE.countDown(); + } +} -- 2.7.4