From 7b9c95d20da5772c6006ff7eb151a26314556093 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Fri, 13 Feb 2015 22:33:52 +0800 Subject: [PATCH] HBASE-13040 Possible failure of TestHMasterRPCException --- .../org/apache/hadoop/hbase/master/HMaster.java | 4 - .../hadoop/hbase/master/MasterRpcServices.java | 1 - .../hbase/master/TestHMasterRPCException.java | 111 +++++++++++++-------- 3 files changed, 67 insertions(+), 49 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 61a1c66..590f36d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -57,14 +57,12 @@ import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.PleaseHoldException; -import org.apache.hadoop.hbase.RegionLocations; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; @@ -109,7 +107,6 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost; import org.apache.hadoop.hbase.procedure.flush.MasterFlushTableProcedureManager; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionServerInfo; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; import org.apache.hadoop.hbase.quotas.MasterQuotaManager; @@ -125,7 +122,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.EncryptionTest; import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HBaseFsckRepair; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.Pair; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 4af53a4..efbf96b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.TableState; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java index 2419918..47f5fd8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHMasterRPCException.java @@ -19,15 +19,18 @@ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.fail; +import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.net.SocketTimeoutException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.ipc.RpcClientFactory; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -36,60 +39,80 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRe import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; import com.google.protobuf.BlockingRpcChannel; import com.google.protobuf.ServiceException; -@Category({MasterTests.class, MediumTests.class}) +@Category({ MasterTests.class, MediumTests.class }) public class TestHMasterRPCException { - @Test - public void testRPCException() throws Exception { - HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - TEST_UTIL.startMiniZKCluster(); - Configuration conf = TEST_UTIL.getConfiguration(); + private static final Log LOG = LogFactory.getLog(TestHMasterRPCException.class); + + private final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + + private HMaster master; + + private RpcClient rpcClient; + + @Before + public void setUp() throws Exception { + Configuration conf = testUtil.getConfiguration(); conf.set(HConstants.MASTER_PORT, "0"); + conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 2000); + testUtil.startMiniZKCluster(); + CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(conf); - HMaster hm = new HMaster(conf, cp); - ServerName sm = hm.getServerName(); - RpcClient rpcClient = RpcClientFactory.createClient(conf, HConstants.CLUSTER_ID_DEFAULT); - try { - int i = 0; - //retry the RPC a few times; we have seen SocketTimeoutExceptions if we - //try to connect too soon. Retry on SocketTimeoutException. - while (i < 20) { - try { - BlockingRpcChannel channel = - rpcClient.createBlockingRpcChannel(sm, User.getCurrent(), 0); - MasterProtos.MasterService.BlockingInterface stub = + ZooKeeperWatcher watcher = testUtil.getZooKeeperWatcher(); + ZKUtil.createWithParents(watcher, watcher.getMasterAddressZNode(), Bytes.toBytes("fake:123")); + master = new HMaster(conf, cp); + rpcClient = RpcClientFactory.createClient(conf, HConstants.CLUSTER_ID_DEFAULT); + } + + @After + public void tearDown() throws IOException { + if (rpcClient != null) { + rpcClient.close(); + } + if (master != null) { + master.stopMaster(); + } + testUtil.shutdownMiniZKCluster(); + } + + @Test + public void testRPCException() throws IOException, InterruptedException, KeeperException { + ServerName sm = master.getServerName(); + boolean fakeZNodeDelete = false; + for (int i = 0; i < 20; i++) { + try { + BlockingRpcChannel channel = rpcClient.createBlockingRpcChannel(sm, User.getCurrent(), 0); + MasterProtos.MasterService.BlockingInterface stub = MasterProtos.MasterService.newBlockingStub(channel); - stub.isMasterRunning(null, IsMasterRunningRequest.getDefaultInstance()); - fail(); - } catch (ServiceException ex) { - IOException ie = ProtobufUtil.getRemoteException(ex); - if (!(ie instanceof SocketTimeoutException)) { - if (ie.getMessage().startsWith("org.apache.hadoop.hbase.ipc." + - "ServerNotRunningYetException: Server is not running yet")) { - // Done. Got the exception we wanted. - System.out.println("Expected exception: " + ie.getMessage()); - return; - } else { - throw ex; - } - } else { - System.err.println("Got SocketTimeoutException. Will retry. "); - } - } catch (Throwable t) { - fail("Unexpected throwable: " + t); + assertTrue(stub.isMasterRunning(null, IsMasterRunningRequest.getDefaultInstance()) + .getIsMasterRunning()); + return; + } catch (ServiceException ex) { + IOException ie = ProtobufUtil.getRemoteException(ex); + // No SocketTimeoutException here. RpcServer is already started after the construction of + // HMaster. + assertTrue(ie.getMessage().startsWith( + "org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet")); + LOG.info("Expected exception: ", ie); + if (!fakeZNodeDelete) { + testUtil.getZooKeeperWatcher().getRecoverableZooKeeper() + .delete(testUtil.getZooKeeperWatcher().getMasterAddressZNode(), -1); + fakeZNodeDelete = true; } - Thread.sleep(100); - i++; } - fail(); - } finally { - rpcClient.close(); + Thread.sleep(1000); } } } \ No newline at end of file -- 1.9.1