[HBASE-7729] TestCatalogTrackerOnCluster.testbadOriginalRootLocation fails occasionally

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 0.94.5, 0.95.0
• Component/s: test
• Hadoop Flags: Reviewed
• Assignee: Lars Hofhansl
• Reporter: Lars Hofhansl
• Created: 31/Jan/13 16:59
• Updated: 26/Feb/13 08:27
• Resolved: 31/Jan/13 22:44

Description

Failure:

java.io.IOException: Shutting down
	at org.apache.hadoop.hbase.MiniHBaseCluster.init(MiniHBaseCluster.java:223)
	at org.apache.hadoop.hbase.MiniHBaseCluster.<init>(MiniHBaseCluster.java:86)
	at org.apache.hadoop.hbase.MiniHBaseCluster.<init>(MiniHBaseCluster.java:77)
	at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniHBaseCluster(HBaseTestingUtility.java:650)
	at org.apache.hadoop.hbase.catalog.TestCatalogTrackerOnCluster.testBadOriginalRootLocation(TestCatalogTrackerOnCluster.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:18)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:24)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:441)
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
	at java.util.concurrent.FutureTask.run(FutureTask.java:138)
	at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
	at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.RuntimeException: Master not initialized after 200 seconds
	at org.apache.hadoop.hbase.util.JVMClusterUtil.startup(JVMClusterUtil.java:206)
	at org.apache.hadoop.hbase.LocalHBaseCluster.startup(LocalHBaseCluster.java:420)
	at org.apache.hadoop.hbase.MiniHBaseCluster.init(MiniHBaseCluster.java:216)
	... 32 more

Likely caused by this:

2013-01-31 04:52:23,064 FATAL [Master:0;hemera.apache.org,52696,1359607882775] master.HMaster(1493): Unhandled exception. Starting shutdown.
org.apache.hadoop.hbase.ipc.HBaseClient$FailedServerException: This server is in the failed servers list: example.org/192.0.43.10:1234
	at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:425)
	at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1124)
	at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:974)
	at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
	at $Proxy19.getProtocolVersion(Unknown Source)
	at org.apache.hadoop.hbase.ipc.WritableRpcEngine.getProxy(WritableRpcEngine.java:138)
	at org.apache.hadoop.hbase.ipc.HBaseRPC.waitForProxy(HBaseRPC.java:208)
	at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1335)
	at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1291)
	at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHRegionConnection(HConnectionManager.java:1278)
	at org.apache.hadoop.hbase.catalog.CatalogTracker.getCachedConnection(CatalogTracker.java:506)
	at org.apache.hadoop.hbase.catalog.CatalogTracker.getRootServerConnection(CatalogTracker.java:343)
	at org.apache.hadoop.hbase.catalog.CatalogTracker.waitForRootServerConnection(CatalogTracker.java:327)
	at org.apache.hadoop.hbase.catalog.CatalogTracker.verifyRootRegionLocation(CatalogTracker.java:599)
	at org.apache.hadoop.hbase.master.HMaster.assignRootAndMeta(HMaster.java:659)
	at org.apache.hadoop.hbase.master.HMaster.finishInitialization(HMaster.java:560)
	at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:376)
	at java.lang.Thread.run(Thread.java:662)
2013-01-31 04:52:23,064 INFO  [Master:0;hemera.apache.org,52696,1359607882775] master.HMaster(1652): Aborting