Error Message test timed out after 30000 milliseconds Stacktrace org.junit.runners.model.TestTimedOutException: test timed out after 30000 milliseconds at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) Standard Output 2017-02-23 21:49:11,638 INFO [main] hbase.HBaseTestingUtility(471): Created new mini-cluster data directory: /xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/dfscluster_613867e3-133f-4437-bcef-017d3b7c5a7b, deleteOnExit=true 2017-02-23 21:49:11,775 INFO [main] zookeeper.MiniZooKeeperCluster(276): Started MiniZooKeeperCluster and ran successful 'stat' on client port=50222 2017-02-23 21:49:11,836 INFO [main] hbase.ResourceChecker(147): before: master.TestMasterNoCluster#testStopDuringStart Thread=9, OpenFileDescriptor=223, MaxFileDescriptor=1024000, SystemLoadAverage=719, ProcessCount=11, AvailableMemoryMB=17498, ConnectionCount=0 2017-02-23 21:49:12,037 DEBUG [Time-limited test] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 21:49:12,291 INFO [Time-limited test] client.ConnectionUtils(104): master/xxx/xxx:0 server-side HConnection retries=350 2017-02-23 21:49:12,459 INFO [Time-limited test] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 21:49:12,470 INFO [Time-limited test] ipc.RpcServer$Listener(608): master/xxx/xxx:0: started 10 reader(s). 2017-02-23 21:49:12,524 WARN [Time-limited test] impl.MetricsConfig(125): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2017-02-23 21:49:12,692 WARN [Time-limited test] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2017-02-23 21:49:12,741 INFO [Time-limited test] hfile.CacheConfig(530): Allocating LruBlockCache size=995.58 MB, blockSize=64 KB 2017-02-23 21:49:12,747 DEBUG [Time-limited test] hfile.CacheConfig(544): Trying to use Internal l2 cache 2017-02-23 21:49:12,748 INFO [Time-limited test] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:12,750 INFO [Time-limited test] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:12,753 INFO [Time-limited test] mob.MobFileCache(121): MobFileCache is initialized, and the cache size is 1000 2017-02-23 21:49:12,885 DEBUG [Time-limited test] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:12,888 DEBUG [Time-limited test] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:12,967 INFO [Time-limited test] zookeeper.RecoverableZooKeeper(120): Process identifier=master:56015 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:13,012 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:560150x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:13,013 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(578): master:56015-0x15a6cf30a960000 connected 2017-02-23 21:49:13,057 DEBUG [Time-limited test] zookeeper.ZKUtil(369): master:56015-0x15a6cf30a960000, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2017-02-23 21:49:13,058 DEBUG [Time-limited test] zookeeper.ZKUtil(369): master:56015-0x15a6cf30a960000, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 21:49:13,062 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 21:49:13,062 INFO [RpcServer.listener,port=56015] ipc.RpcServer$Listener(739): RpcServer.listener,port=56015: starting 2017-02-23 21:49:13,063 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=56015 2017-02-23 21:49:13,063 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=56015 2017-02-23 21:49:13,063 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=56015 2017-02-23 21:49:13,064 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=56015 2017-02-23 21:49:13,064 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=56015 2017-02-23 21:49:13,064 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=56015 2017-02-23 21:49:13,064 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=56015 2017-02-23 21:49:13,065 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=56015 2017-02-23 21:49:13,065 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=56015 2017-02-23 21:49:13,065 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=56015 2017-02-23 21:49:13,065 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=56015 2017-02-23 21:49:13,066 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=56015 2017-02-23 21:49:13,066 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=56015 2017-02-23 21:49:13,066 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=56015 2017-02-23 21:49:13,066 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=56015 2017-02-23 21:49:13,067 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=56015 2017-02-23 21:49:13,067 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=56015 2017-02-23 21:49:13,067 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=56015 2017-02-23 21:49:13,067 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=56015 2017-02-23 21:49:13,067 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=56015 2017-02-23 21:49:13,068 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=56015 2017-02-23 21:49:13,068 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=56015 2017-02-23 21:49:13,068 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=56015 2017-02-23 21:49:13,068 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=56015 2017-02-23 21:49:13,068 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=56015 2017-02-23 21:49:13,069 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=56015 2017-02-23 21:49:13,069 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=56015 2017-02-23 21:49:13,069 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=56015 2017-02-23 21:49:13,069 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=56015 2017-02-23 21:49:13,073 INFO [Time-limited test] master.HMaster(429): hbase.rootdir=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424, hbase.cluster.distributed=false 2017-02-23 21:49:13,079 DEBUG [Time-limited test] impl.BackupManager(136): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2017-02-23 21:49:13,080 DEBUG [Time-limited test] impl.BackupManager(137): Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager 2017-02-23 21:49:13,080 DEBUG [Time-limited test] impl.BackupManager(138): Added master observer: org.apache.hadoop.hbase.backup.master.BackupController 2017-02-23 21:49:13,090 INFO [Time-limited test] master.HMaster(1897): Adding backup master ZNode /hbase/backup-masters/xxx,56015,1487886552596 2017-02-23 21:49:13,147 DEBUG [Time-limited test] zookeeper.ZKUtil(367): master:56015-0x15a6cf30a960000, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/xxx,56015,1487886552596 2017-02-23 21:49:13,149 INFO [Time-limited test] regionserver.HRegionServer(1991): STOPPED: Stopped by Time-limited test 2017-02-23 21:49:13,181 INFO [master/xxx/xxx:0] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x9eb620c connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:13,185 DEBUG [master/xxx/xxx:0-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x9eb620c0x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:13,185 DEBUG [master/xxx/xxx:0-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x9eb620c-0x15a6cf30a960001 connected 2017-02-23 21:49:13,185 INFO [master/xxx/xxx:0] client.ZooKeeperRegistry(107): ClusterId read in ZooKeeper is null 2017-02-23 21:49:13,185 DEBUG [master/xxx/xxx:0] client.ConnectionManager$HConnectionImplementation(892): clusterid came back null, using default default-cluster 2017-02-23 21:49:13,199 DEBUG [master/xxx/xxx:0] ipc.AbstractRpcClient(105): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d40e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2017-02-23 21:49:14,207 INFO [master/xxx/xxx:0] ipc.RpcServer(2248): Stopping server on 56015 2017-02-23 21:49:14,208 INFO [RpcServer.listener,port=56015] ipc.RpcServer$Listener(786): RpcServer.listener,port=56015: stopping 2017-02-23 21:49:14,208 INFO [RpcServer.responder] ipc.RpcServer$Responder(1028): RpcServer.responder: stopped 2017-02-23 21:49:14,268 INFO [master/xxx/xxx:0] regionserver.HRegionServer(1066): stopping server xxx,56015,1487886552596 2017-02-23 21:49:14,268 INFO [RpcServer.responder] ipc.RpcServer$Responder(931): RpcServer.responder: stopping 2017-02-23 21:49:14,268 DEBUG [master/xxx/xxx:0] zookeeper.MetaTableLocator(627): Stopping MetaTableLocator 2017-02-23 21:49:14,269 INFO [master/xxx/xxx:0] client.ConnectionManager$HConnectionImplementation(1687): Closing zookeeper sessionid=0x15a6cf30a960001 2017-02-23 21:49:14,271 DEBUG [master/xxx/xxx:0] ipc.RpcClientImpl(1115): Stopping rpc client 2017-02-23 21:49:14,275 INFO [master/xxx/xxx:0] regionserver.HRegionServer(1094): stopping server xxx,56015,1487886552596; all regions closed. 2017-02-23 21:49:14,276 INFO [master/xxx/xxx:0] hbase.ChoreService(302): Chore service for: xxx,56015,1487886552596 had [] on shutdown 2017-02-23 21:49:14,276 DEBUG [master/xxx/xxx:0] master.HMaster(1243): Stopping service threads 2017-02-23 21:49:14,280 DEBUG [master/xxx/xxx:0] zookeeper.ZKUtil(626): master:56015-0x15a6cf30a960000, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2017-02-23 21:49:14,280 WARN [master/xxx/xxx:0] master.ActiveMasterManager(269): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2017-02-23 21:49:14,280 INFO [master/xxx/xxx:0] ipc.RpcServer(2248): Stopping server on 56015 2017-02-23 21:49:14,284 INFO [master/xxx/xxx:0] zookeeper.RecoverableZooKeeper(188): Node /hbase/rs/xxx,56015,1487886552596 already deleted, retry=false 2017-02-23 21:49:14,287 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:56015-0x15a6cf30a960000, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/xxx,56015,1487886552596 2017-02-23 21:49:14,287 INFO [master/xxx/xxx:0] regionserver.HRegionServer(1138): stopping server xxx,56015,1487886552596; zookeeper connection closed. 2017-02-23 21:49:14,287 INFO [master/xxx/xxx:0] regionserver.HRegionServer(1141): master/xxx/xxx:0 exiting 2017-02-23 21:49:14,289 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=@Before connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:14,292 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before0x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:14,292 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): @Before-0x15a6cf30a960002 connected 2017-02-23 21:49:14,294 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/master-maintenance 2017-02-23 21:49:14,295 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/recovering-regions 2017-02-23 21:49:14,296 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/splitWAL 2017-02-23 21:49:14,297 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/rs 2017-02-23 21:49:14,298 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters 2017-02-23 21:49:14,298 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/region-in-transition 2017-02-23 21:49:14,299 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/draining 2017-02-23 21:49:14,300 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/table 2017-02-23 21:49:14,302 DEBUG [main] zookeeper.ZKUtil(367): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/table-lock 2017-02-23 21:49:14,317 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table-lock 2017-02-23 21:49:14,317 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2017-02-23 21:49:14,317 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table 2017-02-23 21:49:14,317 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining 2017-02-23 21:49:14,318 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/region-in-transition 2017-02-23 21:49:14,318 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2017-02-23 21:49:14,318 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2017-02-23 21:49:14,318 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2017-02-23 21:49:14,318 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/recovering-regions 2017-02-23 21:49:14,319 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2017-02-23 21:49:14,319 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960002, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase 2017-02-23 21:49:14,329 INFO [main] hbase.ResourceChecker(171): after: master.TestMasterNoCluster#testStopDuringStart Thread=14 (was 9) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3188) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: master/xxx/xxx:0-SendThread(localhost:50222) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1229) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1166) Potentially hanging thread: Time-limited test-SendThread(localhost:50222) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1229) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1166) Potentially hanging thread: main-SendThread(localhost:50222) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1229) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1166) - Thread LEAK? -, OpenFileDescriptor=236 (was 223) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1024000 (was 1024000), SystemLoadAverage=719 (was 719), ProcessCount=11 (was 11), AvailableMemoryMB=17373 (was 17498), ConnectionCount=0 (was 0) 2017-02-23 21:49:14,335 INFO [main] hbase.ResourceChecker(147): before: master.TestMasterNoCluster#testNotPullingDeadRegionServerFromZK Thread=14, OpenFileDescriptor=236, MaxFileDescriptor=1024000, SystemLoadAverage=719, ProcessCount=11, AvailableMemoryMB=17373, ConnectionCount=0 2017-02-23 21:49:14,337 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=test.sample,1,101 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:14,340 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): test.sample,1,1010x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:14,340 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): test.sample,1,101-0x15a6cf30a960003 connected 2017-02-23 21:49:14,356 DEBUG [main] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 21:49:14,359 INFO [main] client.ConnectionUtils(104): master/xxx/xxx:0 server-side HConnection retries=3500 2017-02-23 21:49:14,360 INFO [main] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 21:49:14,363 INFO [main] ipc.RpcServer$Listener(608): master/xxx/xxx:0: started 10 reader(s). 2017-02-23 21:49:14,363 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:14,364 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:14,365 DEBUG [main] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:14,367 DEBUG [main] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:14,368 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=master:44237 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:14,370 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:442370x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:14,371 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): master:44237-0x15a6cf30a960004 connected 2017-02-23 21:49:14,372 DEBUG [main] zookeeper.RecoverableZooKeeper(584): Node /hbase already exists 2017-02-23 21:49:14,378 DEBUG [main] zookeeper.ZKUtil(369): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2017-02-23 21:49:14,380 DEBUG [main] zookeeper.ZKUtil(369): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 21:49:14,380 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 21:49:14,380 INFO [RpcServer.listener,port=44237] ipc.RpcServer$Listener(739): RpcServer.listener,port=44237: starting 2017-02-23 21:49:14,380 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=44237 2017-02-23 21:49:14,381 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=44237 2017-02-23 21:49:14,381 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=44237 2017-02-23 21:49:14,382 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=44237 2017-02-23 21:49:14,382 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=44237 2017-02-23 21:49:14,383 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=44237 2017-02-23 21:49:14,383 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=44237 2017-02-23 21:49:14,383 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=44237 2017-02-23 21:49:14,384 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=44237 2017-02-23 21:49:14,384 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=44237 2017-02-23 21:49:14,385 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=44237 2017-02-23 21:49:14,385 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=44237 2017-02-23 21:49:14,385 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=44237 2017-02-23 21:49:14,386 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=44237 2017-02-23 21:49:14,386 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=44237 2017-02-23 21:49:14,386 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=44237 2017-02-23 21:49:14,387 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=44237 2017-02-23 21:49:14,387 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=44237 2017-02-23 21:49:14,388 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=44237 2017-02-23 21:49:14,388 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=44237 2017-02-23 21:49:14,388 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=44237 2017-02-23 21:49:14,389 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=44237 2017-02-23 21:49:14,389 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=44237 2017-02-23 21:49:14,389 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=44237 2017-02-23 21:49:14,390 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=44237 2017-02-23 21:49:14,390 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=44237 2017-02-23 21:49:14,390 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=44237 2017-02-23 21:49:14,391 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=44237 2017-02-23 21:49:14,391 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=44237 2017-02-23 21:49:14,392 INFO [main] master.HMaster(429): hbase.rootdir=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424, hbase.cluster.distributed=false 2017-02-23 21:49:14,392 DEBUG [main] impl.BackupManager(136): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2017-02-23 21:49:14,392 DEBUG [main] impl.BackupManager(137): Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager 2017-02-23 21:49:14,392 DEBUG [main] impl.BackupManager(138): Added master observer: org.apache.hadoop.hbase.backup.master.BackupController 2017-02-23 21:49:14,393 INFO [main] master.HMaster(1897): Adding backup master ZNode /hbase/backup-masters/xxx,44237,1487886554363 2017-02-23 21:49:14,396 DEBUG [main] zookeeper.ZKUtil(367): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/xxx,44237,1487886554363 2017-02-23 21:49:14,398 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2017-02-23 21:49:14,399 DEBUG [xxx:44237.activeMasterManager] zookeeper.ZKUtil(367): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 21:49:14,400 INFO [xxx:44237.activeMasterManager] master.ActiveMasterManager(170): Deleting ZNode for /hbase/backup-masters/xxx,44237,1487886554363 from backup master directory 2017-02-23 21:49:14,400 DEBUG [main-EventThread] zookeeper.ZKUtil(367): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 21:49:14,400 DEBUG [main-EventThread] master.ActiveMasterManager(126): A master is now available 2017-02-23 21:49:14,401 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/xxx,44237,1487886554363 2017-02-23 21:49:14,401 WARN [xxx:44237.activeMasterManager] hbase.ZNodeClearer(58): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2017-02-23 21:49:14,401 INFO [master/xxx/xxx:0] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x5a1057f5 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:14,402 INFO [xxx:44237.activeMasterManager] master.ActiveMasterManager(179): Registered Active Master=xxx,44237,1487886554363 2017-02-23 21:49:14,405 DEBUG [master/xxx/xxx:0-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x5a1057f50x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:14,405 DEBUG [master/xxx/xxx:0-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x5a1057f5-0x15a6cf30a960005 connected 2017-02-23 21:49:14,405 INFO [master/xxx/xxx:0] client.ZooKeeperRegistry(107): ClusterId read in ZooKeeper is null 2017-02-23 21:49:14,405 DEBUG [master/xxx/xxx:0] client.ConnectionManager$HConnectionImplementation(892): clusterid came back null, using default default-cluster 2017-02-23 21:49:14,406 DEBUG [master/xxx/xxx:0] ipc.AbstractRpcClient(105): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc0a9f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2017-02-23 21:49:14,463 INFO [xxx:44237.activeMasterManager] util.FSUtils(763): Created version file at file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424 with version=8 2017-02-23 21:49:14,470 DEBUG [xxx:44237.activeMasterManager] util.FSUtils(915): Created cluster ID file at file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/hbase.id with ID: d50c6cd7-a067-4c2d-bc83-70571cdd8ada 2017-02-23 21:49:14,476 INFO [xxx:44237.activeMasterManager] master.MasterFileSystem(560): BOOTSTRAP: creating hbase:meta region 2017-02-23 21:49:14,479 INFO [xxx:44237.activeMasterManager] regionserver.HRegion(6360): creating HRegion hbase:meta HTD == 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|'}, {NAME => 'info', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '10', TTL => 'FOREVER', MIN_VERSIONS => '0', CACHE_DATA_IN_L1 => 'true', KEEP_DELETED_CELLS => 'FALSE', BLOCKSIZE => '8192', IN_MEMORY => 'false', BLOCKCACHE => 'false'} RootDir = file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424 Table name == hbase:meta 2017-02-23 21:49:14,509 INFO [xxx:44237.activeMasterManager] wal.WALFactory(145): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.DefaultWALProvider 2017-02-23 21:49:14,522 INFO [xxx:44237.activeMasterManager] wal.FSHLog(581): WAL configuration: blocksize=32 MB, rollsize=30.40 MB, prefix=hregion-11028977.default, suffix=, logDir=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/WALs/hregion-11028977, archiveDir=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/oldWALs 2017-02-23 21:49:14,537 INFO [xxx:44237.activeMasterManager] wal.FSHLog(1620): Slow sync cost: 0 ms, current pipeline: [] 2017-02-23 21:49:14,537 INFO [xxx:44237.activeMasterManager] wal.FSHLog(1048): New WAL /xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/WALs/hregion-11028977/hregion-11028977.default.1487886554522 2017-02-23 21:49:14,537 INFO [xxx:44237.activeMasterManager] wal.FSHLog(1657): FileSystem's output stream doesn't support getNumCurrentReplicas; HDFS-826 not available; fsOut=java.io.BufferedOutputStream 2017-02-23 21:49:14,537 INFO [xxx:44237.activeMasterManager] wal.FSHLog(2372): FileSystem's output stream doesn't support getPipeline; not available; fsOut=java.io.BufferedOutputStream 2017-02-23 21:49:14,573 DEBUG [xxx:44237.activeMasterManager] regionserver.HRegion(797): Instantiated hbase:meta,,1.1588230740 2017-02-23 21:49:14,632 INFO [StoreOpener-1588230740-1] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=false, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:14,646 INFO [StoreOpener-1588230740-1] compactions.CompactionConfiguration(133): size [134217728, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, base window in milliseconds 21600000, windows per tier 4,incoming window min 6 2017-02-23 21:49:14,663 DEBUG [StoreOpener-1588230740-1] regionserver.HRegionFileSystem(202): No StoreFiles for: file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/data/hbase/meta/1588230740/info 2017-02-23 21:49:14,669 DEBUG [StoreOpener-1588230740-1] util.ChecksumType$2(70): org.apache.hadoop.util.PureJavaCrc32 available 2017-02-23 21:49:14,670 DEBUG [StoreOpener-1588230740-1] util.ChecksumType$3(113): org.apache.hadoop.util.PureJavaCrc32C available 2017-02-23 21:49:14,680 DEBUG [xxx:44237.activeMasterManager] regionserver.HRegion(4092): Found 0 recovered edits file(s) under file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/data/hbase/meta/1588230740 2017-02-23 21:49:14,691 DEBUG [xxx:44237.activeMasterManager] wal.WALSplitter(720): Wrote region seqId=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/data/hbase/meta/1588230740/recovered.edits/2.seqid to file, newSeqId=2, maxSeqId=0 2017-02-23 21:49:14,691 INFO [xxx:44237.activeMasterManager] regionserver.HRegion(924): Onlined 1588230740; next sequenceid=2 2017-02-23 21:49:14,692 DEBUG [xxx:44237.activeMasterManager] regionserver.HRegion(1453): Closing hbase:meta,,1.1588230740: disabling compactions & flushes 2017-02-23 21:49:14,692 DEBUG [xxx:44237.activeMasterManager] regionserver.HRegion(1480): Updates disabled for region hbase:meta,,1.1588230740 2017-02-23 21:49:14,693 INFO [StoreCloserThread-hbase:meta,,1.1588230740-1] regionserver.HStore(917): Closed info 2017-02-23 21:49:14,694 INFO [xxx:44237.activeMasterManager] regionserver.HRegion(1581): Closed hbase:meta,,1.1588230740 2017-02-23 21:49:14,694 DEBUG [xxx:44237.activeMasterManager] wal.FSHLog(1245): Closing WAL writer in /xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/WALs/hregion-11028977 2017-02-23 21:49:14,696 DEBUG [xxx:44237.activeMasterManager] wal.FSHLog(1203): Moved 1 WAL file(s) to /xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/oldWALs 2017-02-23 21:49:14,697 INFO [xxx:44237.activeMasterManager] wal.FSHLog(1206): Closed WAL: FSHLog hregion-11028977.default:(num 1487886554522) 2017-02-23 21:49:14,718 DEBUG [xxx:44237.activeMasterManager] util.FSTableDescriptors(660): Wrote descriptor into: file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/data/hbase/meta/.tabledesc/.tableinfo.0000000001 2017-02-23 21:49:14,726 DEBUG [xxx:44237.activeMasterManager] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:14,730 DEBUG [xxx:44237.activeMasterManager] coordination.ZKSplitLogManagerCoordination(886): Distributed log replay=false 2017-02-23 21:49:14,733 INFO [xxx:44237.activeMasterManager] coordination.ZKSplitLogManagerCoordination(599): Found 0 orphan tasks and 0 rescan nodes 2017-02-23 21:49:14,734 DEBUG [xxx:44237.activeMasterManager] util.FSTableDescriptors(208): Fetching table descriptors from the filesystem. 2017-02-23 21:49:14,749 INFO [xxx:44237.activeMasterManager] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x2f4b83f5 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:14,753 DEBUG [xxx:44237.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x2f4b83f50x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:14,754 DEBUG [xxx:44237.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x2f4b83f5-0x15a6cf30a960006 connected 2017-02-23 21:49:14,755 DEBUG [xxx:44237.activeMasterManager] ipc.AbstractRpcClient(105): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a899663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2017-02-23 21:49:14,777 INFO [xxx:44237.activeMasterManager] balancer.StochasticLoadBalancer(155): loading config 2017-02-23 21:49:14,789 DEBUG [xxx:44237.activeMasterManager] zookeeper.ZKUtil(369): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/balancer 2017-02-23 21:49:14,791 DEBUG [xxx:44237.activeMasterManager] zookeeper.ZKUtil(369): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/normalizer 2017-02-23 21:49:14,795 DEBUG [xxx:44237.activeMasterManager] zookeeper.ZKUtil(369): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/switch/split 2017-02-23 21:49:14,796 DEBUG [xxx:44237.activeMasterManager] zookeeper.ZKUtil(369): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/switch/merge 2017-02-23 21:49:14,842 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:44237-0x15a6cf30a960004, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2017-02-23 21:49:14,843 INFO [xxx:44237.activeMasterManager] master.HMaster(673): Server active/primary master=xxx,44237,1487886554363, sessionid=0x15a6cf30a960004, setting cluster-up flag (Was=false) 2017-02-23 21:49:14,846 INFO [master/xxx/xxx:0] regionserver.HRegionServer(807): ClusterId : d50c6cd7-a067-4c2d-bc83-70571cdd8ada 2017-02-23 21:49:14,853 INFO [xxx:44237.activeMasterManager] procedure.ProcedureManagerHost(71): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2017-02-23 21:49:14,867 INFO [xxx:44237.activeMasterManager] procedure.ZKProcedureUtil(270): Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort 2017-02-23 21:49:14,870 DEBUG [xxx:44237.activeMasterManager] procedure.ZKProcedureCoordinatorRpcs(238): Starting the controller for procedure member:xxx,44237,1487886554363 2017-02-23 21:49:14,880 INFO [xxx:44237.activeMasterManager] procedure.ZKProcedureUtil(270): Clearing all procedure znodes: /hbase/rolllog-proc/acquired /hbase/rolllog-proc/reached /hbase/rolllog-proc/abort 2017-02-23 21:49:14,882 DEBUG [xxx:44237.activeMasterManager] procedure.ZKProcedureCoordinatorRpcs(238): Starting the controller for procedure member:xxx,44237,1487886554363 2017-02-23 21:49:14,890 INFO [xxx:44237.activeMasterManager] procedure.ZKProcedureUtil(270): Clearing all procedure znodes: /hbase/flush-table-proc/acquired /hbase/flush-table-proc/reached /hbase/flush-table-proc/abort 2017-02-23 21:49:14,892 DEBUG [xxx:44237.activeMasterManager] procedure.ZKProcedureCoordinatorRpcs(238): Starting the controller for procedure member:xxx,44237,1487886554363 2017-02-23 21:49:14,905 INFO [xxx:44237.activeMasterManager] master.ServerManager(452): Registering server=test.sample,1,101 2017-02-23 21:49:15,094 INFO [xxx:44237.activeMasterManager] master.MasterCoprocessorHost(92): System coprocessor loading is enabled 2017-02-23 21:49:15,099 INFO [xxx:44237.activeMasterManager] coprocessor.CoprocessorHost(160): System coprocessor org.apache.hadoop.hbase.backup.master.BackupController was loaded successfully with priority (536870911). 2017-02-23 21:49:15,100 DEBUG [xxx:44237.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_OPEN_REGION-xxx:44237, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:15,100 DEBUG [xxx:44237.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_CLOSE_REGION-xxx:44237, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:15,100 DEBUG [xxx:44237.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_SERVER_OPERATIONS-xxx:44237, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:15,100 DEBUG [xxx:44237.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_META_SERVER_OPERATIONS-xxx:44237, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:15,101 DEBUG [xxx:44237.activeMasterManager] executor.ExecutorService(100): Starting executor service name=M_LOG_REPLAY_OPS-xxx:44237, corePoolSize=10, maxPoolSize=10 2017-02-23 21:49:15,101 DEBUG [xxx:44237.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_TABLE_OPERATIONS-xxx:44237, corePoolSize=1, maxPoolSize=1 2017-02-23 21:49:15,113 INFO [xxx:44237.activeMasterManager] procedure2.ProcedureExecutor(436): Starting procedure executor threads=33 2017-02-23 21:49:15,114 INFO [xxx:44237.activeMasterManager] wal.WALProcedureStore(318): Starting WAL Procedure Store lease recovery 2017-02-23 21:49:15,114 WARN [xxx:44237.activeMasterManager] wal.WALProcedureStore(983): Log directory not found: File file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/MasterProcWALs does not exist 2017-02-23 21:49:15,123 DEBUG [xxx:44237.activeMasterManager] wal.WALProcedureStore(861): Roll new state log: 1 2017-02-23 21:49:15,124 INFO [xxx:44237.activeMasterManager] wal.WALProcedureStore(347): Lease acquired for flushLogId: 1 2017-02-23 21:49:15,124 DEBUG [xxx:44237.activeMasterManager] wal.WALProcedureStore(364): No state logs to replay. 2017-02-23 21:49:15,132 DEBUG [ProcedureExecutorTimeout] procedure2.ProcedureExecutor$CompletedProcedureCleaner(162): No completed procedures to cleanup. 2017-02-23 21:49:15,133 DEBUG [xxx:44237.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2017-02-23 21:49:15,133 DEBUG [xxx:44237.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.master ...[truncated 424471 bytes]... va:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:640) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:627) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: xxx,50493,1487886585484_ChoreService_1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: RpcServer.reader=4,bindAddress=xxx,port=50493 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:640) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:627) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: RpcServer.reader=3,bindAddress=xxx,port=50493 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:640) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:627) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: ProcedureExecutorThread-4 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: ProcedureExecutorThread-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: ProcedureExecutorThread-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: ProcedureExecutorThread-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: ProcedureExecutorThread-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.reader=1,bindAddress=xxx,port=50493 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:640) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:627) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: Time-limited test-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ProcedureExecutorThread-10 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: ProcedureExecutorThread-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: ProcedureExecutorThread-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: ProcedureExecutorThread-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: ProcedureExecutorThread-22 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: ProcedureExecutorThread-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: xxx:50493.activeMasterManager java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.master.ServerManager.waitForRegionServers(ServerManager.java:1043) org.apache.hadoop.hbase.master.ServerManager$$EnhancerByMockitoWithCGLIB$$edcbe467.CGLIB$waitForRegionServers$22() org.apache.hadoop.hbase.master.ServerManager$$EnhancerByMockitoWithCGLIB$$edcbe467$$FastClassByMockitoWithCGLIB$$eec56390.invoke() org.mockito.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:216) org.mockito.internal.creation.AbstractMockitoMethodProxy.invokeSuper(AbstractMockitoMethodProxy.java:10) org.mockito.internal.invocation.realmethod.CGLIBProxyRealMethod.invoke(CGLIBProxyRealMethod.java:22) org.mockito.internal.invocation.realmethod.FilteredCGLIBProxyRealMethod.invoke(FilteredCGLIBProxyRealMethod.java:27) org.mockito.internal.invocation.InvocationImpl.callRealMethod(InvocationImpl.java:112) org.mockito.internal.stubbing.answers.CallsRealMethods.answer(CallsRealMethods.java:36) org.mockito.internal.handler.MockHandlerImpl.handle(MockHandlerImpl.java:93) org.mockito.internal.handler.NullResultGuardian.handle(NullResultGuardian.java:29) org.mockito.internal.handler.InvocationNotifierHandler.handle(InvocationNotifierHandler.java:38) org.mockito.internal.creation.MethodInterceptorFilter.intercept(MethodInterceptorFilter.java:61) org.apache.hadoop.hbase.master.ServerManager$$EnhancerByMockitoWithCGLIB$$edcbe467.waitForRegionServers() org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:763) org.apache.hadoop.hbase.master.HMaster.access$600(HMaster.java:222) org.apache.hadoop.hbase.master.HMaster$2.run(HMaster.java:1927) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: xxx:50493.activeMasterManager-SendThread(localhost:50222) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1125) Potentially hanging thread: Time-limited test-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: RpcServer.reader=0,bindAddress=xxx,port=50493 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:640) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:627) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: ProcedureExecutorThread-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: WALProcedureStoreSyncThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2176) org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.syncLoop(WALProcedureStore.java:583) org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.access$000(WALProcedureStore.java:68) org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore$1.run(WALProcedureStore.java:221) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: RpcServer.reader=8,bindAddress=xxx,port=50493 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:640) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:627) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:722) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: ProcedureExecutorTimeout sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.poll(TimeoutBlockingQueue.java:106) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.timeoutLoop(ProcedureExecutor.java:917) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$300(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$2.run(ProcedureExecutor.java:452) Potentially hanging thread: RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=50493 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303) java.util.concurrent.Semaphore.acquire(Semaphore.java:317) org.apache.hadoop.hbase.ipc.FastPathBalancedQueueRpcExecutor$FastPathHandler.getCallRunner(FastPathBalancedQueueRpcExecutor.java:105) org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:167) Potentially hanging thread: ProcedureExecutorThread-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) Potentially hanging thread: RpcServer.listener,port=50493 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener.run(RpcServer.java:743) Potentially hanging thread: ProcedureExecutorThread-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.master.procedure.MasterProcedureQueue.poll(MasterProcedureQueue.java:117) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execLoop(ProcedureExecutor.java:835) org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$200(ProcedureExecutor.java:77) org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.run(ProcedureExecutor.java:443) - Thread LEAK? -, OpenFileDescriptor=311 (was 250) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1024000 (was 1024000), SystemLoadAverage=765 (was 741) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17198 (was 17260), ConnectionCount=1 (was 1) 2017-02-23 21:50:15,504 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,505 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,506 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,509 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,510 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,510 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method)2017-02-23 21:50:15,511 INFO [main] zookeeper.MiniZooKeeperCluster(319): Shutdown MiniZK cluster with all ZK servers at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,513 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,513 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,514 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,516 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,517 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) Standard Error 2017-02-23 21:49:45,405 INFO [main] hbase.ResourceChecker(171): after: master.TestMasterNoCluster#testNotPullingDeadRegionServerFromZK Thread=21 (was 14) Potentially hanging thread: main-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: xxx:44237.activeMasterManager-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: snapshot-log-cleaner-cache-refresher java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: xxx:44237.activeMasterManager-SendThread(localhost:50222) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1125) Potentially hanging thread: xxx:44237.activeMasterManager-SendThread(localhost:50222) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1229) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1166) Potentially hanging thread: snapshot-hfile-cleaner-cache-refresher java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) - Thread LEAK? -, OpenFileDescriptor=250 (was 236) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1024000 (was 1024000), SystemLoadAverage=741 (was 719) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17261 (was 17373), ConnectionCount=1 (was 0) - ConnectionCount LEAK? - 2017-02-23 21:49:45,409 INFO [main] hbase.ResourceChecker(147): before: master.TestMasterNoCluster#testFailover Thread=21, OpenFileDescriptor=250, MaxFileDescriptor=1024000, SystemLoadAverage=741, ProcessCount=11, AvailableMemoryMB=17260, ConnectionCount=1 2017-02-23 21:49:45,410 INFO [Time-limited test] zookeeper.RecoverableZooKeeper(120): Process identifier=0.example.org,0,1487886585410 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,413 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): 0.example.org,0,14878865854100x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,413 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(578): 0.example.org,0,1487886585410-0x15a6cf30a960009 connected 2017-02-23 21:49:45,443 INFO [Time-limited test] zookeeper.RecoverableZooKeeper(120): Process identifier=1.example.org,1,1487886585410 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,447 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): 1.example.org,1,14878865854100x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,447 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(578): 1.example.org,1,1487886585410-0x15a6cf30a96000a connected 2017-02-23 21:49:45,448 DEBUG [Time-limited test] zookeeper.RecoverableZooKeeper(584): Node /hbase already exists 2017-02-23 21:49:45,453 INFO [Time-limited test] zookeeper.RecoverableZooKeeper(120): Process identifier=2.example.org,2,1487886585410 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,455 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): 2.example.org,2,14878865854100x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,455 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(578): 2.example.org,2,1487886585410-0x15a6cf30a96000b connected 2017-02-23 21:49:45,456 DEBUG [Time-limited test] zookeeper.RecoverableZooKeeper(584): Node /hbase already exists 2017-02-23 21:49:45,460 INFO [Time-limited test] zookeeper.MetaTableLocator(451): Setting hbase:meta region location in ZooKeeper as 0.example.org,0,1487886585410 2017-02-23 21:49:45,467 DEBUG [Time-limited test] zookeeper.MetaTableLocator(463): META region location doesn't exist, create it 2017-02-23 21:49:45,477 DEBUG [Time-limited test] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 21:49:45,480 INFO [Time-limited test] client.ConnectionUtils(104): master/xxx/xxx:0 server-side HConnection retries=35000 2017-02-23 21:49:45,480 INFO [Time-limited test] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 21:49:45,483 INFO [Time-limited test] ipc.RpcServer$Listener(608): master/xxx/xxx:0: started 10 reader(s). 2017-02-23 21:49:45,484 INFO [Time-limited test] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:45,484 INFO [Time-limited test] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 21:49:45,485 DEBUG [Time-limited test] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:45,486 DEBUG [Time-limited test] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:45,487 INFO [Time-limited test] zookeeper.RecoverableZooKeeper(120): Process identifier=master:50493 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,490 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:504930x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,490 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(578): master:50493-0x15a6cf30a96000c connected 2017-02-23 21:49:45,491 DEBUG [Time-limited test] zookeeper.RecoverableZooKeeper(584): Node /hbase already exists 2017-02-23 21:49:45,495 DEBUG [Time-limited test] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2017-02-23 21:49:45,496 DEBUG [Time-limited test] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 21:49:45,497 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 21:49:45,497 INFO [RpcServer.listener,port=50493] ipc.RpcServer$Listener(739): RpcServer.listener,port=50493: starting 2017-02-23 21:49:45,497 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=50493 2017-02-23 21:49:45,497 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=50493 2017-02-23 21:49:45,497 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=50493 2017-02-23 21:49:45,498 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=50493 2017-02-23 21:49:45,498 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=50493 2017-02-23 21:49:45,498 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=50493 2017-02-23 21:49:45,498 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=50493 2017-02-23 21:49:45,499 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=50493 2017-02-23 21:49:45,499 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=50493 2017-02-23 21:49:45,499 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=50493 2017-02-23 21:49:45,499 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=50493 2017-02-23 21:49:45,499 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=50493 2017-02-23 21:49:45,500 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=50493 2017-02-23 21:49:45,500 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=50493 2017-02-23 21:49:45,500 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=50493 2017-02-23 21:49:45,500 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=50493 2017-02-23 21:49:45,500 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=50493 2017-02-23 21:49:45,501 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=50493 2017-02-23 21:49:45,501 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=50493 2017-02-23 21:49:45,501 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=50493 2017-02-23 21:49:45,501 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=50493 2017-02-23 21:49:45,501 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=50493 2017-02-23 21:49:45,502 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=50493 2017-02-23 21:49:45,502 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=50493 2017-02-23 21:49:45,502 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=50493 2017-02-23 21:49:45,502 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=50493 2017-02-23 21:49:45,502 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=50493 2017-02-23 21:49:45,503 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=50493 2017-02-23 21:49:45,503 DEBUG [Time-limited test] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=50493 2017-02-23 21:49:45,503 INFO [Time-limited test] master.HMaster(429): hbase.rootdir=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424, hbase.cluster.distributed=false 2017-02-23 21:49:45,503 DEBUG [Time-limited test] impl.BackupManager(136): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2017-02-23 21:49:45,504 DEBUG [Time-limited test] impl.BackupManager(137): Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager 2017-02-23 21:49:45,504 DEBUG [Time-limited test] impl.BackupManager(138): Added master observer: org.apache.hadoop.hbase.backup.master.BackupController 2017-02-23 21:49:45,504 INFO [Time-limited test] master.HMaster(1897): Adding backup master ZNode /hbase/backup-masters/xxx,50493,1487886585484 2017-02-23 21:49:45,506 DEBUG [Time-limited test] zookeeper.ZKUtil(367): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/xxx,50493,1487886585484 2017-02-23 21:49:45,508 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2017-02-23 21:49:45,508 INFO [master/xxx/xxx:0] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x160a4e66 connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,508 DEBUG [xxx:50493.activeMasterManager] zookeeper.ZKUtil(367): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 21:49:45,510 INFO [xxx:50493.activeMasterManager] master.ActiveMasterManager(170): Deleting ZNode for /hbase/backup-masters/xxx,50493,1487886585484 from backup master directory 2017-02-23 21:49:45,510 DEBUG [Time-limited test-EventThread] zookeeper.ZKUtil(367): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 21:49:45,510 DEBUG [Time-limited test-EventThread] master.ActiveMasterManager(126): A master is now available 2017-02-23 21:49:45,511 DEBUG [master/xxx/xxx:0-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x160a4e660x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,511 DEBUG [master/xxx/xxx:0-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x160a4e66-0x15a6cf30a96000d connected 2017-02-23 21:49:45,512 INFO [master/xxx/xxx:0] client.ZooKeeperRegistry(107): ClusterId read in ZooKeeper is null 2017-02-23 21:49:45,512 DEBUG [master/xxx/xxx:0] client.ConnectionManager$HConnectionImplementation(892): clusterid came back null, using default default-cluster 2017-02-23 21:49:45,512 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/xxx,50493,1487886585484 2017-02-23 21:49:45,512 WARN [xxx:50493.activeMasterManager] hbase.ZNodeClearer(58): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2017-02-23 21:49:45,512 DEBUG [master/xxx/xxx:0] ipc.AbstractRpcClient(105): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5107cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2017-02-23 21:49:45,512 INFO [xxx:50493.activeMasterManager] master.ActiveMasterManager(179): Registered Active Master=xxx,50493,1487886585484 2017-02-23 21:49:45,517 DEBUG [xxx:50493.activeMasterManager] util.FSTableDescriptors(731): Current tableInfoPath = file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/data/hbase/meta/.tabledesc/.tableinfo.0000000001 2017-02-23 21:49:45,519 DEBUG [xxx:50493.activeMasterManager] util.FSTableDescriptors(735): TableInfo already exists.. Skipping creation 2017-02-23 21:49:45,522 DEBUG [xxx:50493.activeMasterManager] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering 2017-02-23 21:49:45,523 DEBUG [xxx:50493.activeMasterManager] coordination.ZKSplitLogManagerCoordination(886): Distributed log replay=false 2017-02-23 21:49:45,524 INFO [xxx:50493.activeMasterManager] coordination.ZKSplitLogManagerCoordination(599): Found 0 orphan tasks and 0 rescan nodes 2017-02-23 21:49:45,525 DEBUG [xxx:50493.activeMasterManager] util.FSTableDescriptors(208): Fetching table descriptors from the filesystem. 2017-02-23 21:49:45,530 INFO [xxx:50493.activeMasterManager] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x604a68b connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,532 DEBUG [xxx:50493.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x604a68b0x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,532 DEBUG [xxx:50493.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x604a68b-0x15a6cf30a96000e connected 2017-02-23 21:49:45,533 DEBUG [xxx:50493.activeMasterManager] ipc.AbstractRpcClient(105): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9122c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2017-02-23 21:49:45,599 INFO [xxx:50493.activeMasterManager] balancer.StochasticLoadBalancer(155): loading config 2017-02-23 21:49:45,600 DEBUG [xxx:50493.activeMasterManager] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/balancer 2017-02-23 21:49:45,600 DEBUG [xxx:50493.activeMasterManager] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/normalizer 2017-02-23 21:49:45,603 DEBUG [xxx:50493.activeMasterManager] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/switch/split 2017-02-23 21:49:45,604 DEBUG [xxx:50493.activeMasterManager] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/switch/merge 2017-02-23 21:49:45,608 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2017-02-23 21:49:45,608 INFO [xxx:50493.activeMasterManager] master.HMaster(673): Server active/primary master=xxx,50493,1487886585484, sessionid=0x15a6cf30a96000c, setting cluster-up flag (Was=false) 2017-02-23 21:49:45,609 INFO [xxx:50493.activeMasterManager] procedure.ProcedureManagerHost(71): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2017-02-23 21:49:45,610 INFO [master/xxx/xxx:0] regionserver.HRegionServer(807): ClusterId : d50c6cd7-a067-4c2d-bc83-70571cdd8ada 2017-02-23 21:49:45,617 INFO [xxx:50493.activeMasterManager] procedure.ZKProcedureUtil(270): Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort 2017-02-23 21:49:45,619 DEBUG [xxx:50493.activeMasterManager] procedure.ZKProcedureCoordinatorRpcs(238): Starting the controller for procedure member:xxx,50493,1487886585484 2017-02-23 21:49:45,625 INFO [xxx:50493.activeMasterManager] procedure.ZKProcedureUtil(270): Clearing all procedure znodes: /hbase/rolllog-proc/acquired /hbase/rolllog-proc/reached /hbase/rolllog-proc/abort 2017-02-23 21:49:45,627 DEBUG [xxx:50493.activeMasterManager] procedure.ZKProcedureCoordinatorRpcs(238): Starting the controller for procedure member:xxx,50493,1487886585484 2017-02-23 21:49:45,635 INFO [xxx:50493.activeMasterManager] procedure.ZKProcedureUtil(270): Clearing all procedure znodes: /hbase/flush-table-proc/acquired /hbase/flush-table-proc/reached /hbase/flush-table-proc/abort 2017-02-23 21:49:45,637 DEBUG [xxx:50493.activeMasterManager] procedure.ZKProcedureCoordinatorRpcs(238): Starting the controller for procedure member:xxx,50493,1487886585484 2017-02-23 21:49:45,637 INFO [xxx:50493.activeMasterManager] master.MasterCoprocessorHost(92): System coprocessor loading is enabled 2017-02-23 21:49:45,638 INFO [xxx:50493.activeMasterManager] coprocessor.CoprocessorHost(160): System coprocessor org.apache.hadoop.hbase.backup.master.BackupController was loaded successfully with priority (536870911). 2017-02-23 21:49:45,638 DEBUG [xxx:50493.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_OPEN_REGION-xxx:50493, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:45,639 DEBUG [xxx:50493.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_CLOSE_REGION-xxx:50493, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:45,639 DEBUG [xxx:50493.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_SERVER_OPERATIONS-xxx:50493, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:45,639 DEBUG [xxx:50493.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_META_SERVER_OPERATIONS-xxx:50493, corePoolSize=5, maxPoolSize=5 2017-02-23 21:49:45,639 DEBUG [xxx:50493.activeMasterManager] executor.ExecutorService(100): Starting executor service name=M_LOG_REPLAY_OPS-xxx:50493, corePoolSize=10, maxPoolSize=10 2017-02-23 21:49:45,640 DEBUG [xxx:50493.activeMasterManager] executor.ExecutorService(100): Starting executor service name=MASTER_TABLE_OPERATIONS-xxx:50493, corePoolSize=1, maxPoolSize=1 2017-02-23 21:49:45,641 INFO [xxx:50493.activeMasterManager] procedure2.ProcedureExecutor(436): Starting procedure executor threads=33 2017-02-23 21:49:45,642 INFO [xxx:50493.activeMasterManager] wal.WALProcedureStore(318): Starting WAL Procedure Store lease recovery 2017-02-23 21:49:45,645 DEBUG [xxx:50493.activeMasterManager] wal.WALProcedureStore(1045): Opening state-log: DeprecatedRawLocalFileStatus{path=file:/xxxhbase/hbase-server/target/test-data/cffcf536-f37c-4a9a-9aac-1246e40cb424/MasterProcWALs/state-00000000000000000001.log; isDirectory=false; length=30; replication=1; blocksize=33554432; modification_time=1487886585000; access_time=1487886555000; owner=; group=; permission=rw-rw-rw-; isSymlink=false} 2017-02-23 21:49:45,656 DEBUG [xxx:50493.activeMasterManager] wal.WALProcedureStore(861): Roll new state log: 2 2017-02-23 21:49:45,657 INFO [xxx:50493.activeMasterManager] wal.WALProcedureStore(347): Lease acquired for flushLogId: 2 2017-02-23 21:49:45,663 DEBUG [ProcedureExecutorTimeout] procedure2.ProcedureExecutor$CompletedProcedureCleaner(162): No completed procedures to cleanup. 2017-02-23 21:49:45,663 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2017-02-23 21:49:45,664 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2017-02-23 21:49:45,666 INFO [xxx:50493.activeMasterManager] zookeeper.RecoverableZooKeeper(120): Process identifier=replicationLogCleaner connecting to ZooKeeper ensemble=localhost:50222 2017-02-23 21:49:45,670 DEBUG [xxx:50493.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(515): replicationLogCleaner0x0, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:49:45,670 DEBUG [xxx:50493.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(578): replicationLogCleaner-0x15a6cf30a96000f connected 2017-02-23 21:49:45,672 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2017-02-23 21:49:45,673 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotLogCleaner 2017-02-23 21:49:45,674 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2017-02-23 21:49:45,675 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2017-02-23 21:49:45,675 DEBUG [xxx:50493.activeMasterManager] cleaner.CleanerChore(91): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2017-02-23 21:49:45,712 INFO [xxx:50493.activeMasterManager] master.ServerManager(1037): Waiting for region servers count to settle; currently checked in 0, slept for 0 ms, expecting minimum of 1, maximum of 2147483647, timeout of 30000 ms, interval of 1500 ms. 2017-02-23 21:49:45,712 INFO [Time-limited test] master.ServerManager(452): Registering server=0.example.org,0,1487886585410 2017-02-23 21:49:45,715 INFO [Time-limited test] master.ServerManager(452): Registering server=1.example.org,1,1487886585410 2017-02-23 21:49:45,715 INFO [Time-limited test] master.ServerManager(452): Registering server=2.example.org,2,1487886585410 2017-02-23 21:49:45,716 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:45,718 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:45,763 INFO [xxx:50493.activeMasterManager] master.ServerManager(1037): Waiting for region servers count to settle; currently checked in 3, slept for 51 ms, expecting minimum of 1, maximum of 2147483647, timeout of 30000 ms, interval of 1500 ms. 2017-02-23 21:49:45,818 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:45,820 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:45,920 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:45,921 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,022 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,023 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,123 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,125 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,225 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,226 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,327 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,328 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,428 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,430 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,530 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,531 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,631 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,632 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,733 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,734 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,834 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,836 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:46,936 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:46,937 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,037 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,038 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,139 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,140 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,240 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,242 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,271 INFO [xxx:50493.activeMasterManager] master.ServerManager(1037): Waiting for region servers count to settle; currently checked in 3, slept for 1559 ms, expecting minimum of 1, maximum of 2147483647, timeout of 30000 ms, interval of 1500 ms. 2017-02-23 21:49:47,342 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,343 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,443 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,445 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,545 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,546 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,646 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,648 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,748 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,749 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,849 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,851 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:47,951 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:47,952 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,053 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,054 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,154 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,156 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,256 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,257 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,358 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,359 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,459 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,461 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,561 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,563 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,663 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,664 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,765 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,766 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,779 INFO [xxx:50493.activeMasterManager] master.ServerManager(1037): Waiting for region servers count to settle; currently checked in 3, slept for 3067 ms, expecting minimum of 1, maximum of 2147483647, timeout of 30000 ms, interval of 1500 ms. 2017-02-23 21:49:48,866 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,868 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:48,968 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:48,969 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,069 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,071 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,171 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,173 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,273 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,275 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,375 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,377 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,477 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,478 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,579 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,580 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,680 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,681 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,782 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:49:49,783 DEBUG [Time-limited test] zookeeper.ZKUtil(716): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 because node does not exist (not necessarily an error) 2017-02-23 21:49:49,884 DEBUG [Time-limited test] zook ...[truncated 234674 chars]... lientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,459 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,462 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,462 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,464 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted2017-02-23 21:50:15,464 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/reached 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/flush-table-proc at java.lang.Thread.sleep(Native Method) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/abort at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc/acquired at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/reached at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/online-snapshot at java.lang.reflect.Method.invoke(Method.java:601) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/abort at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) 2017-02-23 21:50:15,465 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot/acquired at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) 2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rolllog-proc/reached at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) 2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rolllog-proc at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) 2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rolllog-proc/abort at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) 2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rolllog-proc/acquired at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166)2017-02-23 21:50:15,466 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs at java.lang.Thread.run(Thread.java:722)2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/flush-table-proc 2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2017-02-23 21:50:15,466 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,466 INFO [Time-limited test-EventThread] zookeeper.RegionServerTracker(118): RegionServer ephemeral node deleted, processing expiration [rs] 2017-02-23 21:50:15,466 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs 2017-02-23 21:50:15,467 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,468 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)2017-02-23 21:50:15,467 ERROR [Time-limited test-EventThread] zookeeper.ClientCnxn$EventThread(536): Error while calling watcher java.lang.IllegalArgumentException: Not a host:port pair: rs at org.apache.hadoop.hbase.util.Addressing.parseHostname(Addressing.java:68) at org.apache.hadoop.hbase.ServerName.(ServerName.java:129) at org.apache.hadoop.hbase.ServerName.valueOf(ServerName.java:181) at org.apache.hadoop.hbase.ServerName.parseServerName(ServerName.java:359) at org.apache.hadoop.hbase.zookeeper.RegionServerTracker.nodeDeleted(RegionServerTracker.java:120) at org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.process(ZooKeeperWatcher.java:539) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:534) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:510) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) 2017-02-23 21:50:15,467 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/recovering-regions at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) 2017-02-23 21:50:15,469 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,468 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): 0.example.org,0,1487886585410-0x15a6cf30a960009, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server 2017-02-23 21:50:15,469 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,469 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitWAL 2017-02-23 21:50:15,470 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,471 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) 2017-02-23 21:50:15,470 DEBUG [Time-limited test-EventThread] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) 2017-02-23 21:50:15,470 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)2017-02-23 21:50:15,471 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/online-snapshot at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601)2017-02-23 21:50:15,472 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/hbaseid at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)2017-02-23 21:50:15,472 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298)2017-02-23 21:50:15,472 DEBUG [Time-limited test-EventThread] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)2017-02-23 21:50:15,472 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table-lock at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,472 DEBUG [Time-limited test-EventThread] master.ActiveMasterManager(130): No master available. Notifying waiting threads 2017-02-23 21:50:15,472 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,472 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2017-02-23 21:50:15,473 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,473 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted 2017-02-23 21:50:15,473 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 2017-02-23 21:50:15,473 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/region-in-transition at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) 2017-02-23 21:50:15,474 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) 2017-02-23 21:50:15,474 DEBUG [Time-limited test-EventThread] zookeeper.ZKUtil(369): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) 2017-02-23 21:50:15,474 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/table at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) 2017-02-23 21:50:15,475 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/draining at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,475 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/switch 2017-02-23 21:50:15,475 ERROR [Time-limited test-EventThread] zookeeper.ClientCnxn$EventThread(536): Error while calling watcher java.lang.ArrayIndexOutOfBoundsException: 1 at org.apache.hadoop.hbase.ServerName.parsePort(ServerName.java:149) at org.apache.hadoop.hbase.ServerName.(ServerName.java:124) at org.apache.hadoop.hbase.ServerName.valueOf(ServerName.java:172) at org.apache.hadoop.hbase.zookeeper.DrainingServerTracker.nodeDeleted(DrainingServerTracker.java:100) at org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.process(ZooKeeperWatcher.java:539) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:534) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:510) 2017-02-23 21:50:15,475 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,476 DEBUG [Time-limited test-EventThread] zookeeper.ZooKeeperWatcher(515): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance 2017-02-23 21:50:15,475 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters 2017-02-23 21:50:15,476 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,477 DEBUG [Time-limited test-EventThread] zookeeper.ZKUtil(458): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to list children of znode /hbase/master-maintenance because node does not exist (not an error) 2017-02-23 21:50:15,476 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rolllog-proc 2017-02-23 21:50:15,477 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted 2017-02-23 21:50:15,477 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master-maintenance at java.lang.Thread.sleep(Native Method) 2017-02-23 21:50:15,478 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/meta-region-server at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246)2017-02-23 21:50:15,478 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): @Before-0x15a6cf30a960010, quorum=localhost:50222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,480 DEBUG [Time-limited test] zookeeper.ZKAssign(805): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Transitioning 1588230740 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2017-02-23 21:50:15,480 WARN [Time-limited test] zookeeper.ZKUtil(724): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Unable to get data of znode /hbase/region-in-transition/1588230740 java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) 2017-02-23 21:50:15,481 DEBUG [Time-limited test] zookeeper.ZooKeeperWatcher(650): master:50493-0x15a6cf30a96000c, quorum=localhost:50222, baseZNode=/hbase Received InterruptedException, doing nothing here java.lang.InterruptedException at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1386) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1153) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getData(RecoverableZooKeeper.java:354) at org.apache.hadoop.hbase.zookeeper.ZKUtil.getDataNoWatch(ZKUtil.java:712) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:814) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNode(ZKAssign.java:795) at org.apache.hadoop.hbase.zookeeper.ZKAssign.transitionNodeClosed(ZKAssign.java:566) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:244) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:298) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:292) at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) at java.util.concurrent.FutureTask.run(FutureTask.java:166) at java.lang.Thread.run(Thread.java:722) java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:146) at org.apache.hadoop.hbase.master.TestMasterNoCluster.testFailover(TestMasterNoCluster.java:246) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:601) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)