Process Thread Dump: 411 active threads Thread 979237 (IPC Client (1127292223) connection to /10.91.209.20:50020 from blk_1074251769_511324): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.FilterInputStream.read(FilterInputStream.java:133) java.io.FilterInputStream.read(FilterInputStream.java:133) org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:512) java.io.BufferedInputStream.fill(BufferedInputStream.java:235) java.io.BufferedInputStream.read(BufferedInputStream.java:254) java.io.DataInputStream.readInt(DataInputStream.java:387) org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1055) org.apache.hadoop.ipc.Client$Connection.run(Client.java:950) Thread 979140 (hedgedRead-1751): State: TIMED_WAITING Blocked count: 0 Waited count: 1 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 979121 (ResponseProcessor for block BP-664460170-10.91.209.35-1423743007662:blk_1074251774_511329): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.io.FilterInputStream.read(FilterInputStream.java:83) java.io.FilterInputStream.read(FilterInputStream.java:83) org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2107) org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176) org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:810) Thread 979120 (DataStreamer for file /hbase/data/default/lab_hyuga_zero/0d9a85c8031128ca8cfb28730e7a2e25/.tmp/f6e88fa019ac486badfcdb5a7d47fb66 block BP-664460170-10.91.209.35-1423743007662:blk_1074251774_511329): State: TIMED_WAITING Blocked count: 1082 Waited count: 1085 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:512) Thread 979118 (IPC Client (1127292223) connection to hbase-left-slave4.hyuga.local/10.91.209.17:60020 from hbase): State: TIMED_WAITING Blocked count: 0 Waited count: 1 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.RpcClient$Connection.waitForWork(RpcClient.java:678) org.apache.hadoop.hbase.ipc.RpcClient$Connection.run(RpcClient.java:726) Thread 978810 (ResponseProcessor for block BP-664460170-10.91.209.35-1423743007662:blk_1074251769_511324): State: RUNNABLE Blocked count: 145 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) java.io.FilterInputStream.read(FilterInputStream.java:83) java.io.FilterInputStream.read(FilterInputStream.java:83) org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2107) org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176) org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:810) Thread 978809 (DataStreamer for file /hbase/WALs/hbase-left-slave3.hyuga.local,60020,1439967287714/hbase-left-slave3.hyuga.local%2C60020%2C1439967287714.1440038688799 block BP-664460170-10.91.209.35-1423743007662:blk_1074251769_511324): State: TIMED_WAITING Blocked count: 50145 Waited count: 50012 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:512) Thread 977845 (hedgedRead-1749): State: TIMED_WAITING Blocked count: 1 Waited count: 4704 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 977844 (hedgedRead-1748): State: TIMED_WAITING Blocked count: 3 Waited count: 3092 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 976629 (RMI TCP Connection(4874)-10.91.214.36): State: RUNNABLE Blocked count: 0 Waited count: 6 Stack: java.net.SocketInputStream.socketRead0(Native Method) java.net.SocketInputStream.read(SocketInputStream.java:152) java.net.SocketInputStream.read(SocketInputStream.java:122) java.io.BufferedInputStream.fill(BufferedInputStream.java:235) java.io.BufferedInputStream.read(BufferedInputStream.java:254) java.io.FilterInputStream.read(FilterInputStream.java:83) sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:538) sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:811) sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:670) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 975614 (hedgedRead-1747): State: TIMED_WAITING Blocked count: 3 Waited count: 11465 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 975613 (hedgedRead-1746): State: TIMED_WAITING Blocked count: 3 Waited count: 8997 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 974906 (hedgedRead-1745): State: TIMED_WAITING Blocked count: 7 Waited count: 7536 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 973659 (IPC Client (1127292223) connection to hbase-left-master1.hyuga.local/10.91.209.35:8020 from hbase): State: TIMED_WAITING Blocked count: 11274 Waited count: 11244 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:904) org.apache.hadoop.ipc.Client$Connection.run(Client.java:949) Thread 973609 (RMI TCP Connection(4873)-10.91.214.36): State: RUNNABLE Blocked count: 0 Waited count: 10 Stack: java.net.SocketInputStream.socketRead0(Native Method) java.net.SocketInputStream.read(SocketInputStream.java:152) java.net.SocketInputStream.read(SocketInputStream.java:122) java.io.BufferedInputStream.fill(BufferedInputStream.java:235) java.io.BufferedInputStream.read(BufferedInputStream.java:254) java.io.FilterInputStream.read(FilterInputStream.java:83) sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:538) sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:811) sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:670) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 969422 (IPC Client (1127292223) connection to hbase-left-master3.hyuga.local/10.91.209.36:60000 from hbase): State: TIMED_WAITING Blocked count: 190 Waited count: 191 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.RpcClient$Connection.waitForWork(RpcClient.java:678) org.apache.hadoop.hbase.ipc.RpcClient$Connection.run(RpcClient.java:726) Thread 964737 (hedgedRead-1733): State: TIMED_WAITING Blocked count: 14 Waited count: 31302 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 955038 (hedgedRead-1724): State: RUNNABLE Blocked count: 6 Waited count: 53057 Stack: sun.nio.ch.FileDispatcherImpl.pread0(Native Method) sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:52) sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:220) sun.nio.ch.IOUtil.read(IOUtil.java:192) sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:699) sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:684) org.apache.hadoop.hdfs.BlockReaderLocal.fillBuffer(BlockReaderLocal.java:311) org.apache.hadoop.hdfs.BlockReaderLocal.fillDataBuf(BlockReaderLocal.java:439) org.apache.hadoop.hdfs.BlockReaderLocal.readWithBounceBuffer(BlockReaderLocal.java:574) org.apache.hadoop.hdfs.BlockReaderLocal.read(BlockReaderLocal.java:538) org.apache.hadoop.hdfs.BlockReaderUtil.readAll(BlockReaderUtil.java:32) org.apache.hadoop.hdfs.BlockReaderLocal.readAll(BlockReaderLocal.java:627) org.apache.hadoop.hdfs.DFSInputStream.actualGetFromOneDataNode(DFSInputStream.java:1044) org.apache.hadoop.hdfs.DFSInputStream.access$100(DFSInputStream.java:83) org.apache.hadoop.hdfs.DFSInputStream$2.call(DFSInputStream.java:993) org.apache.hadoop.hdfs.DFSInputStream$2.call(DFSInputStream.java:988) java.util.concurrent.FutureTask.run(FutureTask.java:262) java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) java.util.concurrent.FutureTask.run(FutureTask.java:262) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) Thread 952200 (hedgedRead-1720): State: TIMED_WAITING Blocked count: 13 Waited count: 60294 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 946409 (IPC Parameter Sending Thread #48): State: TIMED_WAITING Blocked count: 10 Waited count: 98469 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 627491 (IPC Client (1127292223) connection to hbase-right-slave4.hyuga.local/10.91.211.217:60020 from hbase): State: TIMED_WAITING Blocked count: 85610 Waited count: 85600 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.RpcClient$Connection.waitForWork(RpcClient.java:678) org.apache.hadoop.hbase.ipc.RpcClient$Connection.run(RpcClient.java:726) Thread 340113 (IPC Client (1127292223) connection to hbase-right-slave3.hyuga.local/10.91.211.215:60020 from hbase): State: TIMED_WAITING Blocked count: 155441 Waited count: 155414 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.RpcClient$Connection.waitForWork(RpcClient.java:678) org.apache.hadoop.hbase.ipc.RpcClient$Connection.run(RpcClient.java:726) Thread 340111 (IPC Client (1127292223) connection to hbase-right-slave2.hyuga.local/10.91.211.216:60020 from hbase): State: TIMED_WAITING Blocked count: 154840 Waited count: 154817 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.RpcClient$Connection.waitForWork(RpcClient.java:678) org.apache.hadoop.hbase.ipc.RpcClient$Connection.run(RpcClient.java:726) Thread 340110 (IPC Client (1127292223) connection to hbase-right-slave1.hyuga.local/10.91.211.214:60020 from hbase): State: TIMED_WAITING Blocked count: 154899 Waited count: 154883 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.RpcClient$Connection.waitForWork(RpcClient.java:678) org.apache.hadoop.hbase.ipc.RpcClient$Connection.run(RpcClient.java:726) Thread 153030 (RMI Scheduler(0)): State: TIMED_WAITING Blocked count: 0 Waited count: 4300 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 8904 (RS_CLOSE_REGION-hbase-left-slave3:60020-2): State: WAITING Blocked count: 22 Waited count: 39 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4a75a7c8 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 4116 (RS_CLOSE_REGION-hbase-left-slave3:60020-1): State: WAITING Blocked count: 3 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4a75a7c8 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 4060 (RS_CLOSE_REGION-hbase-left-slave3:60020-0): State: WAITING Blocked count: 46 Waited count: 76 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4a75a7c8 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 3301 (ReplicationExecutor-0): State: WAITING Blocked count: 6 Waited count: 10 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@41276243 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 572 (ShortCircuitCache_SlotReleaser): State: WAITING Blocked count: 9 Waited count: 1780587 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@62534476 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 548 (RS_OPEN_REGION-hbase-left-slave3:60020-2): State: WAITING Blocked count: 35 Waited count: 54 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@48b3d168 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 497 (RS_OPEN_REGION-hbase-left-slave3:60020-1): State: WAITING Blocked count: 50 Waited count: 76 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@48b3d168 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 464 (New I/O worker #1): State: RUNNABLE Blocked count: 9 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) com.cloudera.cmf.event.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) com.cloudera.cmf.event.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 463 (AvroEventStorePublisher-NettyThread-boss_1): State: WAITING Blocked count: 0 Waited count: 1 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@452963c0 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 453 (EventStorePublisherWithRetry-0): State: WAITING Blocked count: 2 Waited count: 2412 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@68295543 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 451 (regionserver60020-smallCompactions-1439967337740): State: RUNNABLE Blocked count: 6481 Waited count: 9272 Stack: org.apache.hadoop.hbase.regionserver.StoreFile$Writer.appendGeneralBloomfilter(StoreFile.java:865) org.apache.hadoop.hbase.regionserver.StoreFile$Writer.append(StoreFile.java:906) org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:242) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:75) org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:121) org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1113) org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1483) org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner.run(CompactSplitThread.java:478) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 447 (RS_OPEN_REGION-hbase-left-slave3:60020-0): State: WAITING Blocked count: 46 Waited count: 74 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@48b3d168 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 446 (ShortCircuitCache_Cleaner): State: TIMED_WAITING Blocked count: 0 Waited count: 959 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 400 (SplitLogWorker-hbase-left-slave3.hyuga.local,60020,1439967287714-EventThread): State: WAITING Blocked count: 0 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@28f4ef11 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Thread 399 (SplitLogWorker-hbase-left-slave3.hyuga.local,60020,1439967287714-SendThread(hbase-left-master3.hyuga.local:2181)): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Thread 397 (SplitLogWorker-hbase-left-slave3.hyuga.local,60020,1439967287714): State: TIMED_WAITING Blocked count: 5 Waited count: 14287 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:261) org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:205) java.lang.Thread.run(Thread.java:745) Thread 396 (ReplicationRpcServer.handler=2,queue=0,port=60020): State: WAITING Blocked count: 0 Waited count: 1 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a58133e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 395 (ReplicationRpcServer.handler=1,queue=0,port=60020): State: WAITING Blocked count: 0 Waited count: 1 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a58133e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 394 (ReplicationRpcServer.handler=0,queue=0,port=60020): State: WAITING Blocked count: 0 Waited count: 1 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a58133e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 393 (PriorityRpcServer.handler=9,queue=0,port=60020): State: WAITING Blocked count: 0 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 392 (PriorityRpcServer.handler=8,queue=0,port=60020): State: WAITING Blocked count: 0 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 391 (PriorityRpcServer.handler=7,queue=0,port=60020): State: WAITING Blocked count: 3 Waited count: 8 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 390 (PriorityRpcServer.handler=6,queue=0,port=60020): State: WAITING Blocked count: 3 Waited count: 8 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 389 (PriorityRpcServer.handler=5,queue=0,port=60020): State: WAITING Blocked count: 3 Waited count: 8 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 388 (PriorityRpcServer.handler=4,queue=0,port=60020): State: WAITING Blocked count: 3 Waited count: 8 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 387 (PriorityRpcServer.handler=3,queue=0,port=60020): State: WAITING Blocked count: 3 Waited count: 8 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 386 (PriorityRpcServer.handler=2,queue=0,port=60020): State: WAITING Blocked count: 3 Waited count: 8 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 385 (PriorityRpcServer.handler=1,queue=0,port=60020): State: WAITING Blocked count: 6 Waited count: 13 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 384 (PriorityRpcServer.handler=0,queue=0,port=60020): State: WAITING Blocked count: 9 Waited count: 20 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2c578a32 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 383 (RW.default.scanRpcServer.handler=299,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 382 (RW.default.scanRpcServer.handler=298,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 381 (RW.default.scanRpcServer.handler=297,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 380 (RW.default.scanRpcServer.handler=296,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 379 (RW.default.scanRpcServer.handler=295,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 378 (RW.default.scanRpcServer.handler=294,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 377 (RW.default.scanRpcServer.handler=293,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 376 (RW.default.scanRpcServer.handler=292,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 375 (RW.default.scanRpcServer.handler=291,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 374 (RW.default.scanRpcServer.handler=290,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 373 (RW.default.scanRpcServer.handler=289,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 372 (RW.default.scanRpcServer.handler=288,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 371 (RW.default.scanRpcServer.handler=287,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 370 (RW.default.scanRpcServer.handler=286,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 369 (RW.default.scanRpcServer.handler=285,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 368 (RW.default.scanRpcServer.handler=284,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 367 (RW.default.scanRpcServer.handler=283,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 366 (RW.default.scanRpcServer.handler=282,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 365 (RW.default.scanRpcServer.handler=281,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 364 (RW.default.scanRpcServer.handler=280,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 363 (RW.default.scanRpcServer.handler=279,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 362 (RW.default.scanRpcServer.handler=278,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 361 (RW.default.scanRpcServer.handler=277,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 360 (RW.default.scanRpcServer.handler=276,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 359 (RW.default.scanRpcServer.handler=275,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 358 (RW.default.scanRpcServer.handler=274,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 357 (RW.default.scanRpcServer.handler=273,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 356 (RW.default.scanRpcServer.handler=272,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 355 (RW.default.scanRpcServer.handler=271,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 354 (RW.default.scanRpcServer.handler=270,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 353 (RW.default.scanRpcServer.handler=269,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 352 (RW.default.scanRpcServer.handler=268,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 351 (RW.default.scanRpcServer.handler=267,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 350 (RW.default.scanRpcServer.handler=266,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 349 (RW.default.scanRpcServer.handler=265,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 348 (RW.default.scanRpcServer.handler=264,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 347 (RW.default.scanRpcServer.handler=263,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 346 (RW.default.scanRpcServer.handler=262,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 345 (RW.default.scanRpcServer.handler=261,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 344 (RW.default.scanRpcServer.handler=260,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 343 (RW.default.scanRpcServer.handler=259,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 342 (RW.default.scanRpcServer.handler=258,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 341 (RW.default.scanRpcServer.handler=257,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 340 (RW.default.scanRpcServer.handler=256,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 339 (RW.default.scanRpcServer.handler=255,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 338 (RW.default.scanRpcServer.handler=254,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 337 (RW.default.scanRpcServer.handler=253,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 336 (RW.default.scanRpcServer.handler=252,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 335 (RW.default.scanRpcServer.handler=251,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 334 (RW.default.scanRpcServer.handler=250,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 333 (RW.default.scanRpcServer.handler=249,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 332 (RW.default.scanRpcServer.handler=248,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 331 (RW.default.scanRpcServer.handler=247,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 330 (RW.default.scanRpcServer.handler=246,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 329 (RW.default.scanRpcServer.handler=245,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 328 (RW.default.scanRpcServer.handler=244,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 327 (RW.default.scanRpcServer.handler=243,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 326 (RW.default.scanRpcServer.handler=242,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 325 (RW.default.scanRpcServer.handler=241,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 324 (RW.default.scanRpcServer.handler=240,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 323 (RW.default.scanRpcServer.handler=239,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 322 (RW.default.scanRpcServer.handler=238,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 321 (RW.default.scanRpcServer.handler=237,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 320 (RW.default.scanRpcServer.handler=236,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 319 (RW.default.scanRpcServer.handler=235,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 318 (RW.default.scanRpcServer.handler=234,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 317 (RW.default.scanRpcServer.handler=233,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 316 (RW.default.scanRpcServer.handler=232,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 315 (RW.default.scanRpcServer.handler=231,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 314 (RW.default.scanRpcServer.handler=230,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 313 (RW.default.scanRpcServer.handler=229,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 312 (RW.default.scanRpcServer.handler=228,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 311 (RW.default.scanRpcServer.handler=227,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 310 (RW.default.scanRpcServer.handler=226,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 309 (RW.default.scanRpcServer.handler=225,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 308 (RW.default.scanRpcServer.handler=224,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 307 (RW.default.scanRpcServer.handler=223,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 306 (RW.default.scanRpcServer.handler=222,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 305 (RW.default.scanRpcServer.handler=221,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 304 (RW.default.scanRpcServer.handler=220,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 303 (RW.default.scanRpcServer.handler=219,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 5 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 302 (RW.default.scanRpcServer.handler=218,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 301 (RW.default.scanRpcServer.handler=217,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 300 (RW.default.scanRpcServer.handler=216,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 299 (RW.default.scanRpcServer.handler=215,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 298 (RW.default.scanRpcServer.handler=214,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 297 (RW.default.scanRpcServer.handler=213,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 296 (RW.default.scanRpcServer.handler=212,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 295 (RW.default.scanRpcServer.handler=211,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 294 (RW.default.scanRpcServer.handler=210,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 293 (RW.default.scanRpcServer.handler=209,queue=29,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2731ef5c Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 292 (org.apache.hadoop.hdfs.PeerCache@2ac40a46): State: TIMED_WAITING Blocked count: 5 Waited count: 23803 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:250) org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:41) org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:119) java.lang.Thread.run(Thread.java:745) Thread 291 (RW.default.scanRpcServer.handler=208,queue=28,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3f59951e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 290 (RW.default.scanRpcServer.handler=207,queue=27,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@124d0510 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 289 (RW.default.scanRpcServer.handler=206,queue=26,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e8766d2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 288 (RW.default.scanRpcServer.handler=205,queue=25,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6b6fa5e3 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 287 (RW.default.scanRpcServer.handler=204,queue=24,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6ef4112f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 286 (RW.default.scanRpcServer.handler=203,queue=23,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59945d91 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 285 (RW.default.scanRpcServer.handler=202,queue=22,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1b8b0889 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 284 (RW.default.scanRpcServer.handler=201,queue=21,port=60020): State: WAITING Blocked count: 0 Waited count: 7 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@44dd687f Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 283 (RW.default.scanRpcServer.handler=200,queue=20,port=60020): State: WAITING Blocked count: 0 Waited count: 6 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@23f78d2b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:218) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 282 (RW.default.readRpcServer.handler=199,queue=10,port=60020): State: BLOCKED Blocked count: 184866 Waited count: 8853467 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 201 (RW.default.readRpcServer.handler=118,queue=19,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 281 (RW.default.readRpcServer.handler=198,queue=19,port=60020): State: BLOCKED Blocked count: 202216 Waited count: 9704851 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 249 (RW.default.readRpcServer.handler=166,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 280 (RW.default.readRpcServer.handler=197,queue=18,port=60020): State: BLOCKED Blocked count: 202961 Waited count: 9701042 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 249 (RW.default.readRpcServer.handler=166,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 279 (RW.default.readRpcServer.handler=196,queue=17,port=60020): State: BLOCKED Blocked count: 202923 Waited count: 9705512 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 194 (RW.default.readRpcServer.handler=111,queue=12,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 278 (RW.default.readRpcServer.handler=195,queue=16,port=60020): State: RUNNABLE Blocked count: 202958 Waited count: 9696883 Stack: java.lang.Object.hashCode(Native Method) java.util.concurrent.ConcurrentHashMap.hash(ConcurrentHashMap.java:333) java.util.concurrent.ConcurrentHashMap.remove(ConcurrentHashMap.java:1175) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.close(HRegion.java:4191) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4876) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 277 (RW.default.readRpcServer.handler=194,queue=15,port=60020): State: BLOCKED Blocked count: 202854 Waited count: 9700271 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 192 (RW.default.readRpcServer.handler=109,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 276 (RW.default.readRpcServer.handler=193,queue=14,port=60020): State: BLOCKED Blocked count: 202231 Waited count: 9701870 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 240 (RW.default.readRpcServer.handler=157,queue=18,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 275 (RW.default.readRpcServer.handler=192,queue=13,port=60020): State: BLOCKED Blocked count: 203980 Waited count: 9700976 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 182 (RW.default.readRpcServer.handler=99,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 274 (RW.default.readRpcServer.handler=191,queue=12,port=60020): State: BLOCKED Blocked count: 202768 Waited count: 9703723 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 182 (RW.default.readRpcServer.handler=99,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 273 (RW.default.readRpcServer.handler=190,queue=11,port=60020): State: BLOCKED Blocked count: 202807 Waited count: 9704287 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 240 (RW.default.readRpcServer.handler=157,queue=18,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 272 (RW.default.readRpcServer.handler=189,queue=10,port=60020): State: BLOCKED Blocked count: 185552 Waited count: 8855664 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 240 (RW.default.readRpcServer.handler=157,queue=18,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 271 (RW.default.readRpcServer.handler=188,queue=19,port=60020): State: BLOCKED Blocked count: 202543 Waited count: 9703927 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 220 (RW.default.readRpcServer.handler=137,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 270 (RW.default.readRpcServer.handler=187,queue=18,port=60020): State: BLOCKED Blocked count: 202496 Waited count: 9698269 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 224 (RW.default.readRpcServer.handler=141,queue=12,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 269 (RW.default.readRpcServer.handler=186,queue=17,port=60020): State: BLOCKED Blocked count: 202375 Waited count: 9703938 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 250 (RW.default.readRpcServer.handler=167,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 268 (RW.default.readRpcServer.handler=185,queue=16,port=60020): State: BLOCKED Blocked count: 203018 Waited count: 9693916 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 224 (RW.default.readRpcServer.handler=141,queue=12,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 267 (RW.default.readRpcServer.handler=184,queue=15,port=60020): State: BLOCKED Blocked count: 204028 Waited count: 9702639 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 250 (RW.default.readRpcServer.handler=167,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 266 (RW.default.readRpcServer.handler=183,queue=14,port=60020): State: BLOCKED Blocked count: 202867 Waited count: 9701458 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 250 (RW.default.readRpcServer.handler=167,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 265 (RW.default.readRpcServer.handler=182,queue=13,port=60020): State: BLOCKED Blocked count: 203062 Waited count: 9701724 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 210 (RW.default.readRpcServer.handler=127,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 264 (RW.default.readRpcServer.handler=181,queue=12,port=60020): State: BLOCKED Blocked count: 203851 Waited count: 9701112 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 210 (RW.default.readRpcServer.handler=127,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3175) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29925) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 263 (RW.default.readRpcServer.handler=180,queue=11,port=60020): State: BLOCKED Blocked count: 202695 Waited count: 9698435 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 162 (RW.default.writeRpcServer.handler=79,queue=9,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 262 (RW.default.readRpcServer.handler=179,queue=10,port=60020): State: BLOCKED Blocked count: 185654 Waited count: 8855413 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 235 (RW.default.readRpcServer.handler=152,queue=13,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 261 (RW.default.readRpcServer.handler=178,queue=19,port=60020): State: BLOCKED Blocked count: 202823 Waited count: 9702430 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 235 (RW.default.readRpcServer.handler=152,queue=13,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 260 (RW.default.readRpcServer.handler=177,queue=18,port=60020): State: BLOCKED Blocked count: 204511 Waited count: 9700364 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 235 (RW.default.readRpcServer.handler=152,queue=13,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020): State: BLOCKED Blocked count: 203130 Waited count: 9702063 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 235 (RW.default.readRpcServer.handler=152,queue=13,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 258 (RW.default.readRpcServer.handler=175,queue=16,port=60020): State: BLOCKED Blocked count: 202180 Waited count: 9696366 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 190 (RW.default.readRpcServer.handler=107,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 257 (RW.default.readRpcServer.handler=174,queue=15,port=60020): State: BLOCKED Blocked count: 203438 Waited count: 9703289 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 190 (RW.default.readRpcServer.handler=107,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 256 (RW.default.readRpcServer.handler=173,queue=14,port=60020): State: RUNNABLE Blocked count: 203204 Waited count: 9704093 Stack: java.lang.Object.hashCode(Native Method) java.util.concurrent.ConcurrentHashMap.hash(ConcurrentHashMap.java:333) java.util.concurrent.ConcurrentHashMap.put(ConcurrentHashMap.java:1125) java.util.Collections$SetFromMap.add(Collections.java:3903) org.apache.hadoop.hbase.regionserver.HStore.addChangedReaderObserver(HStore.java:1035) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:167) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 255 (RW.default.readRpcServer.handler=172,queue=13,port=60020): State: BLOCKED Blocked count: 202328 Waited count: 9702144 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 243 (RW.default.readRpcServer.handler=160,queue=11,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 254 (RW.default.readRpcServer.handler=171,queue=12,port=60020): State: BLOCKED Blocked count: 202332 Waited count: 9704238 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 219 (RW.default.readRpcServer.handler=136,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 253 (RW.default.readRpcServer.handler=170,queue=11,port=60020): State: RUNNABLE Blocked count: 203460 Waited count: 9699084 Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3906) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3901) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4873) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 252 (RW.default.readRpcServer.handler=169,queue=10,port=60020): State: BLOCKED Blocked count: 185032 Waited count: 8855089 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 263 (RW.default.readRpcServer.handler=180,queue=11,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 251 (RW.default.readRpcServer.handler=168,queue=19,port=60020): State: BLOCKED Blocked count: 203943 Waited count: 9703295 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 219 (RW.default.readRpcServer.handler=136,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 250 (RW.default.readRpcServer.handler=167,queue=18,port=60020): State: BLOCKED Blocked count: 202629 Waited count: 9703436 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 199 (RW.default.readRpcServer.handler=116,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 249 (RW.default.readRpcServer.handler=166,queue=17,port=60020): State: BLOCKED Blocked count: 202728 Waited count: 9701013 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 199 (RW.default.readRpcServer.handler=116,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 248 (RW.default.readRpcServer.handler=165,queue=16,port=60020): State: RUNNABLE Blocked count: 203253 Waited count: 9697821 Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 247 (RW.default.readRpcServer.handler=164,queue=15,port=60020): State: BLOCKED Blocked count: 202413 Waited count: 9703265 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 199 (RW.default.readRpcServer.handler=116,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 246 (RW.default.readRpcServer.handler=163,queue=14,port=60020): State: BLOCKED Blocked count: 204090 Waited count: 9703808 Blocked on java.util.concurrent.ConcurrentHashMap@173ab990 Blocked by 202 (RW.default.readRpcServer.handler=119,queue=10,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 245 (RW.default.readRpcServer.handler=162,queue=13,port=60020): State: BLOCKED Blocked count: 203507 Waited count: 9700715 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 237 (RW.default.readRpcServer.handler=154,queue=15,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 244 (RW.default.readRpcServer.handler=161,queue=12,port=60020): State: BLOCKED Blocked count: 203206 Waited count: 9702010 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 124 (RW.default.writeRpcServer.handler=42,queue=2,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 243 (RW.default.readRpcServer.handler=160,queue=11,port=60020): State: BLOCKED Blocked count: 202576 Waited count: 9703269 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 214 (RW.default.readRpcServer.handler=131,queue=12,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 242 (RW.default.readRpcServer.handler=159,queue=10,port=60020): State: BLOCKED Blocked count: 184651 Waited count: 8855320 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 214 (RW.default.readRpcServer.handler=131,queue=12,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 241 (RW.default.readRpcServer.handler=158,queue=19,port=60020): State: BLOCKED Blocked count: 203446 Waited count: 9702494 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 214 (RW.default.readRpcServer.handler=131,queue=12,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 240 (RW.default.readRpcServer.handler=157,queue=18,port=60020): State: BLOCKED Blocked count: 202407 Waited count: 9700457 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 117 (RW.default.writeRpcServer.handler=35,queue=5,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 239 (RW.default.readRpcServer.handler=156,queue=17,port=60020): State: BLOCKED Blocked count: 202055 Waited count: 9701981 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 117 (RW.default.writeRpcServer.handler=35,queue=5,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 238 (RW.default.readRpcServer.handler=155,queue=16,port=60020): State: BLOCKED Blocked count: 203089 Waited count: 9696532 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 200 (RW.default.readRpcServer.handler=117,queue=18,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 237 (RW.default.readRpcServer.handler=154,queue=15,port=60020): State: BLOCKED Blocked count: 203133 Waited count: 9699046 Blocked on java.util.concurrent.ConcurrentHashMap@173ab990 Blocked by 89 (RW.default.writeRpcServer.handler=7,queue=7,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 236 (RW.default.readRpcServer.handler=153,queue=14,port=60020): State: BLOCKED Blocked count: 202771 Waited count: 9703118 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 255 (RW.default.readRpcServer.handler=172,queue=13,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 235 (RW.default.readRpcServer.handler=152,queue=13,port=60020): State: BLOCKED Blocked count: 202982 Waited count: 9703277 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 142 (RW.default.writeRpcServer.handler=60,queue=0,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 234 (RW.default.readRpcServer.handler=151,queue=12,port=60020): State: BLOCKED Blocked count: 201901 Waited count: 9702244 Blocked on java.util.concurrent.ConcurrentHashMap@4f708e84 Blocked by 270 (RW.default.readRpcServer.handler=187,queue=18,port=60020) Stack: org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3836) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 233 (RW.default.readRpcServer.handler=150,queue=11,port=60020): State: BLOCKED Blocked count: 202848 Waited count: 9700840 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 95 (RW.default.writeRpcServer.handler=13,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 232 (RW.default.readRpcServer.handler=149,queue=10,port=60020): State: BLOCKED Blocked count: 185402 Waited count: 8854074 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 95 (RW.default.writeRpcServer.handler=13,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 231 (RW.default.readRpcServer.handler=148,queue=19,port=60020): State: BLOCKED Blocked count: 202795 Waited count: 9705122 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 95 (RW.default.writeRpcServer.handler=13,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 230 (RW.default.readRpcServer.handler=147,queue=18,port=60020): State: BLOCKED Blocked count: 203834 Waited count: 9698453 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 103 (RW.default.writeRpcServer.handler=21,queue=1,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 229 (RW.default.readRpcServer.handler=146,queue=17,port=60020): State: BLOCKED Blocked count: 203077 Waited count: 9699574 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 103 (RW.default.writeRpcServer.handler=21,queue=1,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 228 (RW.default.readRpcServer.handler=145,queue=16,port=60020): State: BLOCKED Blocked count: 202615 Waited count: 9696929 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 103 (RW.default.writeRpcServer.handler=21,queue=1,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 227 (RW.default.readRpcServer.handler=144,queue=15,port=60020): State: BLOCKED Blocked count: 202521 Waited count: 9704200 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 103 (RW.default.writeRpcServer.handler=21,queue=1,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 226 (RW.default.readRpcServer.handler=143,queue=14,port=60020): State: BLOCKED Blocked count: 204259 Waited count: 9703964 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 225 (RW.default.readRpcServer.handler=142,queue=13,port=60020): State: BLOCKED Blocked count: 202415 Waited count: 9705394 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 224 (RW.default.readRpcServer.handler=141,queue=12,port=60020): State: BLOCKED Blocked count: 202212 Waited count: 9702356 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 223 (RW.default.readRpcServer.handler=140,queue=11,port=60020): State: BLOCKED Blocked count: 202420 Waited count: 9698212 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 222 (RW.default.readRpcServer.handler=139,queue=10,port=60020): State: BLOCKED Blocked count: 185025 Waited count: 8853157 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 221 (RW.default.readRpcServer.handler=138,queue=19,port=60020): State: BLOCKED Blocked count: 203136 Waited count: 9702552 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 220 (RW.default.readRpcServer.handler=137,queue=18,port=60020): State: BLOCKED Blocked count: 203334 Waited count: 9700111 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 168 (RW.default.writeRpcServer.handler=85,queue=5,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 219 (RW.default.readRpcServer.handler=136,queue=17,port=60020): State: BLOCKED Blocked count: 203877 Waited count: 9718763 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 168 (RW.default.writeRpcServer.handler=85,queue=5,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 218 (RW.default.readRpcServer.handler=135,queue=16,port=60020): State: BLOCKED Blocked count: 202991 Waited count: 9702096 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 168 (RW.default.writeRpcServer.handler=85,queue=5,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 217 (RW.default.readRpcServer.handler=134,queue=15,port=60020): State: BLOCKED Blocked count: 203678 Waited count: 9704356 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 125 (RW.default.writeRpcServer.handler=43,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 216 (RW.default.readRpcServer.handler=133,queue=14,port=60020): State: RUNNABLE Blocked count: 202909 Waited count: 9702116 Stack: java.lang.Object.hashCode(Native Method) java.util.concurrent.ConcurrentHashMap.hash(ConcurrentHashMap.java:333) java.util.concurrent.ConcurrentHashMap.put(ConcurrentHashMap.java:1125) java.util.Collections$SetFromMap.add(Collections.java:3903) org.apache.hadoop.hbase.regionserver.HStore.addChangedReaderObserver(HStore.java:1035) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:167) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 215 (RW.default.readRpcServer.handler=132,queue=13,port=60020): State: BLOCKED Blocked count: 203598 Waited count: 9701239 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 125 (RW.default.writeRpcServer.handler=43,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 214 (RW.default.readRpcServer.handler=131,queue=12,port=60020): State: RUNNABLE Blocked count: 202424 Waited count: 9702648 Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 213 (RW.default.readRpcServer.handler=130,queue=11,port=60020): State: BLOCKED Blocked count: 203255 Waited count: 9699845 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 212 (RW.default.readRpcServer.handler=129,queue=10,port=60020): State: BLOCKED Blocked count: 185141 Waited count: 8854183 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 211 (RW.default.readRpcServer.handler=128,queue=19,port=60020): State: BLOCKED Blocked count: 202650 Waited count: 9705669 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 210 (RW.default.readRpcServer.handler=127,queue=18,port=60020): State: BLOCKED Blocked count: 202324 Waited count: 9700941 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 209 (RW.default.readRpcServer.handler=126,queue=17,port=60020): State: BLOCKED Blocked count: 202169 Waited count: 9704070 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 208 (RW.default.readRpcServer.handler=125,queue=16,port=60020): State: BLOCKED Blocked count: 203988 Waited count: 9699645 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 259 (RW.default.readRpcServer.handler=176,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 207 (RW.default.readRpcServer.handler=124,queue=15,port=60020): State: BLOCKED Blocked count: 202488 Waited count: 9700661 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 242 (RW.default.readRpcServer.handler=159,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 206 (RW.default.readRpcServer.handler=123,queue=14,port=60020): State: RUNNABLE Blocked count: 202951 Waited count: 9703391 Stack: java.util.Hashtable.get(Hashtable.java:433) java.util.Properties.getProperty(Properties.java:951) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) Thread 205 (RW.default.readRpcServer.handler=122,queue=13,port=60020): State: BLOCKED Blocked count: 202531 Waited count: 9704258 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 242 (RW.default.readRpcServer.handler=159,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 204 (RW.default.readRpcServer.handler=121,queue=12,port=60020): State: BLOCKED Blocked count: 202854 Waited count: 9704012 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 242 (RW.default.readRpcServer.handler=159,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 203 (RW.default.readRpcServer.handler=120,queue=11,port=60020): State: BLOCKED Blocked count: 203606 Waited count: 9704723 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 242 (RW.default.readRpcServer.handler=159,queue=10,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 202 (RW.default.readRpcServer.handler=119,queue=10,port=60020): State: BLOCKED Blocked count: 184980 Waited count: 8856141 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 140 (RW.default.writeRpcServer.handler=58,queue=8,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 201 (RW.default.readRpcServer.handler=118,queue=19,port=60020): State: BLOCKED Blocked count: 202123 Waited count: 9706079 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 229 (RW.default.readRpcServer.handler=146,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 200 (RW.default.readRpcServer.handler=117,queue=18,port=60020): State: BLOCKED Blocked count: 202823 Waited count: 9703264 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 229 (RW.default.readRpcServer.handler=146,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 199 (RW.default.readRpcServer.handler=116,queue=17,port=60020): State: BLOCKED Blocked count: 202869 Waited count: 9703051 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 229 (RW.default.readRpcServer.handler=146,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 198 (RW.default.readRpcServer.handler=115,queue=16,port=60020): State: BLOCKED Blocked count: 201987 Waited count: 9696192 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 229 (RW.default.readRpcServer.handler=146,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 197 (RW.default.readRpcServer.handler=114,queue=15,port=60020): State: BLOCKED Blocked count: 202683 Waited count: 9702604 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 213 (RW.default.readRpcServer.handler=130,queue=11,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 196 (RW.default.readRpcServer.handler=113,queue=14,port=60020): State: BLOCKED Blocked count: 203444 Waited count: 9704450 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 213 (RW.default.readRpcServer.handler=130,queue=11,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 195 (RW.default.readRpcServer.handler=112,queue=13,port=60020): State: BLOCKED Blocked count: 202544 Waited count: 9704245 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 213 (RW.default.readRpcServer.handler=130,queue=11,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 194 (RW.default.readRpcServer.handler=111,queue=12,port=60020): State: BLOCKED Blocked count: 202402 Waited count: 9704941 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 213 (RW.default.readRpcServer.handler=130,queue=11,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 193 (RW.default.readRpcServer.handler=110,queue=11,port=60020): State: BLOCKED Blocked count: 202420 Waited count: 9703841 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 223 (RW.default.readRpcServer.handler=140,queue=11,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 192 (RW.default.readRpcServer.handler=109,queue=10,port=60020): State: BLOCKED Blocked count: 185133 Waited count: 8854379 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 223 (RW.default.readRpcServer.handler=140,queue=11,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 191 (RW.default.readRpcServer.handler=108,queue=19,port=60020): State: BLOCKED Blocked count: 202210 Waited count: 9706634 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 247 (RW.default.readRpcServer.handler=164,queue=15,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 190 (RW.default.readRpcServer.handler=107,queue=18,port=60020): State: BLOCKED Blocked count: 203041 Waited count: 9702444 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 247 (RW.default.readRpcServer.handler=164,queue=15,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 189 (RW.default.readRpcServer.handler=106,queue=17,port=60020): State: BLOCKED Blocked count: 202176 Waited count: 9703658 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 176 (RW.default.writeRpcServer.handler=93,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 188 (RW.default.readRpcServer.handler=105,queue=16,port=60020): State: BLOCKED Blocked count: 202408 Waited count: 9702041 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 176 (RW.default.writeRpcServer.handler=93,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 187 (RW.default.readRpcServer.handler=104,queue=15,port=60020): State: BLOCKED Blocked count: 202732 Waited count: 9701849 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 176 (RW.default.writeRpcServer.handler=93,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 186 (RW.default.readRpcServer.handler=103,queue=14,port=60020): State: BLOCKED Blocked count: 202950 Waited count: 9706614 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 176 (RW.default.writeRpcServer.handler=93,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 185 (RW.default.readRpcServer.handler=102,queue=13,port=60020): State: BLOCKED Blocked count: 202611 Waited count: 9705724 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 180 (RW.default.writeRpcServer.handler=97,queue=7,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 184 (RW.default.readRpcServer.handler=101,queue=12,port=60020): State: BLOCKED Blocked count: 202500 Waited count: 9700623 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 145 (RW.default.writeRpcServer.handler=63,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegionServer.scan(HRegionServer.java:3175) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29925) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 183 (RW.default.readRpcServer.handler=100,queue=11,port=60020): State: RUNNABLE Blocked count: 202857 Waited count: 9703248 Stack: java.lang.Object.hashCode(Native Method) java.util.concurrent.ConcurrentHashMap.hash(ConcurrentHashMap.java:333) java.util.concurrent.ConcurrentHashMap.remove(ConcurrentHashMap.java:1175) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.close(HRegion.java:4191) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4876) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 182 (RW.default.readRpcServer.handler=99,queue=10,port=60020): State: BLOCKED Blocked count: 185069 Waited count: 8856085 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 145 (RW.default.writeRpcServer.handler=63,queue=3,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4847) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2918) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29921) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) Thread 181 (RW.default.writeRpcServer.handler=98,queue=8,port=60020): State: WAITING Blocked count: 15034893 Waited count: 15360973 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@414a76ff Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 180 (RW.default.writeRpcServer.handler=97,queue=7,port=60020): State: RUNNABLE Blocked count: 15025398 Waited count: 15346469 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 179 (RW.default.writeRpcServer.handler=96,queue=6,port=60020): State: WAITING Blocked count: 15063754 Waited count: 15382159 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 178 (RW.default.writeRpcServer.handler=95,queue=5,port=60020): State: BLOCKED Blocked count: 15071770 Waited count: 15390300 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 261 (RW.default.readRpcServer.handler=178,queue=19,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 177 (RW.default.writeRpcServer.handler=94,queue=4,port=60020): State: WAITING Blocked count: 15058697 Waited count: 15379391 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3a089142 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 176 (RW.default.writeRpcServer.handler=93,queue=3,port=60020): State: WAITING Blocked count: 15008761 Waited count: 15330014 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 175 (RW.default.writeRpcServer.handler=92,queue=2,port=60020): State: WAITING Blocked count: 15006010 Waited count: 15325819 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 174 (RW.default.writeRpcServer.handler=91,queue=1,port=60020): State: WAITING Blocked count: 14972363 Waited count: 15292315 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 173 (RW.default.writeRpcServer.handler=90,queue=0,port=60020): State: WAITING Blocked count: 15032973 Waited count: 15352438 Waiting on java.lang.Object@70bd3343 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 172 (RW.default.writeRpcServer.handler=89,queue=9,port=60020): State: WAITING Blocked count: 16735363 Waited count: 17072183 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 171 (RW.default.writeRpcServer.handler=88,queue=8,port=60020): State: WAITING Blocked count: 15026248 Waited count: 15347188 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 170 (RW.default.writeRpcServer.handler=87,queue=7,port=60020): State: WAITING Blocked count: 15057090 Waited count: 15376070 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 169 (RW.default.writeRpcServer.handler=86,queue=6,port=60020): State: WAITING Blocked count: 15051268 Waited count: 15369575 Waiting on java.lang.Object@70bd3343 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 168 (RW.default.writeRpcServer.handler=85,queue=5,port=60020): State: WAITING Blocked count: 15047299 Waited count: 15367861 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 167 (RW.default.writeRpcServer.handler=84,queue=4,port=60020): State: WAITING Blocked count: 15025273 Waited count: 15345915 Waiting on java.lang.Object@7b2d07ac Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 166 (RW.default.writeRpcServer.handler=83,queue=3,port=60020): State: WAITING Blocked count: 15059559 Waited count: 15378737 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 165 (RW.default.writeRpcServer.handler=82,queue=2,port=60020): State: WAITING Blocked count: 15030383 Waited count: 15350022 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 164 (RW.default.writeRpcServer.handler=81,queue=1,port=60020): State: BLOCKED Blocked count: 15010233 Waited count: 15331298 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 208 (RW.default.readRpcServer.handler=125,queue=16,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 163 (RW.default.writeRpcServer.handler=80,queue=0,port=60020): State: WAITING Blocked count: 15015751 Waited count: 15336416 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 162 (RW.default.writeRpcServer.handler=79,queue=9,port=60020): State: WAITING Blocked count: 16684072 Waited count: 17019566 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 161 (RW.default.writeRpcServer.handler=78,queue=8,port=60020): State: BLOCKED Blocked count: 15060718 Waited count: 15381426 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 269 (RW.default.readRpcServer.handler=186,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 160 (RW.default.writeRpcServer.handler=77,queue=7,port=60020): State: WAITING Blocked count: 15013923 Waited count: 15332781 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 159 (RW.default.writeRpcServer.handler=76,queue=6,port=60020): State: WAITING Blocked count: 15037223 Waited count: 15357022 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 158 (RW.default.writeRpcServer.handler=75,queue=5,port=60020): State: WAITING Blocked count: 15042127 Waited count: 15362772 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@41b0bc07 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 157 (RW.default.writeRpcServer.handler=74,queue=4,port=60020): State: WAITING Blocked count: 15046131 Waited count: 15364727 Waiting on java.lang.Object@70ba99f6 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 156 (RW.default.writeRpcServer.handler=73,queue=3,port=60020): State: WAITING Blocked count: 15024133 Waited count: 15343529 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 154 (RW.default.writeRpcServer.handler=72,queue=2,port=60020): State: WAITING Blocked count: 14988472 Waited count: 15309858 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 153 (RW.default.writeRpcServer.handler=71,queue=1,port=60020): State: WAITING Blocked count: 15046701 Waited count: 15367559 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 152 (RW.default.writeRpcServer.handler=70,queue=0,port=60020): State: BLOCKED Blocked count: 15037251 Waited count: 15357825 Blocked on java.util.concurrent.atomic.AtomicLong@71a486e9 Blocked by 145 (RW.default.writeRpcServer.handler=63,queue=3,port=60020) Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 151 (RW.default.writeRpcServer.handler=69,queue=9,port=60020): State: WAITING Blocked count: 16699242 Waited count: 17037582 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 150 (RW.default.writeRpcServer.handler=68,queue=8,port=60020): State: WAITING Blocked count: 15012140 Waited count: 15334811 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 149 (RW.default.writeRpcServer.handler=67,queue=7,port=60020): State: WAITING Blocked count: 15052133 Waited count: 15372028 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 148 (RW.default.writeRpcServer.handler=66,queue=6,port=60020): State: WAITING Blocked count: 15035264 Waited count: 15355595 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 147 (RW.default.writeRpcServer.handler=65,queue=5,port=60020): State: WAITING Blocked count: 15041282 Waited count: 15360413 Waiting on java.lang.Object@7b2d07ac Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 146 (RW.default.writeRpcServer.handler=64,queue=4,port=60020): State: WAITING Blocked count: 15077310 Waited count: 15397940 Waiting on java.lang.Object@70ba99f6 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 145 (RW.default.writeRpcServer.handler=63,queue=3,port=60020): State: WAITING Blocked count: 15033778 Waited count: 15354226 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 144 (RW.default.writeRpcServer.handler=62,queue=2,port=60020): State: WAITING Blocked count: 14992104 Waited count: 15314580 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 143 (RW.default.writeRpcServer.handler=61,queue=1,port=60020): State: WAITING Blocked count: 15028131 Waited count: 15348331 Waiting on java.lang.Object@70ba99f6 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 142 (RW.default.writeRpcServer.handler=60,queue=0,port=60020): State: WAITING Blocked count: 15024304 Waited count: 15343114 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 141 (RW.default.writeRpcServer.handler=59,queue=9,port=60020): State: RUNNABLE Blocked count: 16770726 Waited count: 17105496 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 140 (RW.default.writeRpcServer.handler=58,queue=8,port=60020): State: WAITING Blocked count: 14955278 Waited count: 15277020 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 139 (RW.default.writeRpcServer.handler=57,queue=7,port=60020): State: WAITING Blocked count: 15049625 Waited count: 15369386 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 138 (RW.default.writeRpcServer.handler=56,queue=6,port=60020): State: WAITING Blocked count: 15062831 Waited count: 15383677 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 137 (RW.default.writeRpcServer.handler=55,queue=5,port=60020): State: BLOCKED Blocked count: 15037569 Waited count: 15355472 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 239 (RW.default.readRpcServer.handler=156,queue=17,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 136 (RW.default.writeRpcServer.handler=54,queue=4,port=60020): State: WAITING Blocked count: 14973690 Waited count: 15297043 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 135 (RW.default.writeRpcServer.handler=53,queue=3,port=60020): State: WAITING Blocked count: 15043263 Waited count: 15361969 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 134 (RW.default.writeRpcServer.handler=52,queue=2,port=60020): State: WAITING Blocked count: 15031215 Waited count: 15352513 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 133 (RW.default.writeRpcServer.handler=51,queue=1,port=60020): State: WAITING Blocked count: 15017909 Waited count: 15337686 Waiting on java.lang.Object@70bd3343 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 132 (RW.default.writeRpcServer.handler=50,queue=0,port=60020): State: WAITING Blocked count: 15132034 Waited count: 15447252 Waiting on java.lang.Object@70bd3343 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 131 (RW.default.writeRpcServer.handler=49,queue=9,port=60020): State: BLOCKED Blocked count: 16676443 Waited count: 17013123 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 276 (RW.default.readRpcServer.handler=193,queue=14,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 130 (RW.default.writeRpcServer.handler=48,queue=8,port=60020): State: RUNNABLE Blocked count: 15019116 Waited count: 15339489 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 129 (RW.default.writeRpcServer.handler=47,queue=7,port=60020): State: BLOCKED Blocked count: 15075126 Waited count: 15394107 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 276 (RW.default.readRpcServer.handler=193,queue=14,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 128 (RW.default.writeRpcServer.handler=46,queue=6,port=60020): State: WAITING Blocked count: 15028811 Waited count: 15349468 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 127 (RW.default.writeRpcServer.handler=45,queue=5,port=60020): State: RUNNABLE Blocked count: 15037802 Waited count: 15357766 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 126 (RW.default.writeRpcServer.handler=44,queue=4,port=60020): State: WAITING Blocked count: 15041833 Waited count: 15360452 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3a089142 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 125 (RW.default.writeRpcServer.handler=43,queue=3,port=60020): State: RUNNABLE Blocked count: 15081651 Waited count: 15401898 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 124 (RW.default.writeRpcServer.handler=42,queue=2,port=60020): State: WAITING Blocked count: 15038347 Waited count: 15356640 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 123 (RW.default.writeRpcServer.handler=41,queue=1,port=60020): State: RUNNABLE Blocked count: 15030859 Waited count: 15351631 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 122 (RW.default.writeRpcServer.handler=40,queue=0,port=60020): State: RUNNABLE Blocked count: 14999235 Waited count: 15319581 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 121 (RW.default.writeRpcServer.handler=39,queue=9,port=60020): State: WAITING Blocked count: 16640861 Waited count: 16977595 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@cbd092a Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 120 (RW.default.writeRpcServer.handler=38,queue=8,port=60020): State: RUNNABLE Blocked count: 14986348 Waited count: 15307868 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 119 (RW.default.writeRpcServer.handler=37,queue=7,port=60020): State: WAITING Blocked count: 15061847 Waited count: 15378257 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 118 (RW.default.writeRpcServer.handler=36,queue=6,port=60020): State: WAITING Blocked count: 15055732 Waited count: 15373239 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 117 (RW.default.writeRpcServer.handler=35,queue=5,port=60020): State: WAITING Blocked count: 15034177 Waited count: 15355087 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 116 (RW.default.writeRpcServer.handler=34,queue=4,port=60020): State: RUNNABLE Blocked count: 15068379 Waited count: 15387187 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 115 (RW.default.writeRpcServer.handler=33,queue=3,port=60020): State: WAITING Blocked count: 14982883 Waited count: 15304340 Waiting on java.lang.Object@70bd3343 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 114 (RW.default.writeRpcServer.handler=32,queue=2,port=60020): State: WAITING Blocked count: 14995473 Waited count: 15315102 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 113 (RW.default.writeRpcServer.handler=31,queue=1,port=60020): State: BLOCKED Blocked count: 15009164 Waited count: 15328554 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 197 (RW.default.readRpcServer.handler=114,queue=15,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 112 (RW.default.writeRpcServer.handler=30,queue=0,port=60020): State: BLOCKED Blocked count: 15095145 Waited count: 15416092 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 197 (RW.default.readRpcServer.handler=114,queue=15,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 111 (RW.default.writeRpcServer.handler=29,queue=9,port=60020): State: WAITING Blocked count: 16732939 Waited count: 17066769 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 110 (RW.default.writeRpcServer.handler=28,queue=8,port=60020): State: TIMED_WAITING Blocked count: 15017984 Waited count: 15337866 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ExecutorCompletionService.poll(ExecutorCompletionService.java:202) org.apache.hadoop.hdfs.DFSInputStream.hedgedFetchBlockByteRange(DFSInputStream.java:1128) org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1302) org.apache.hadoop.fs.FSDataInputStream.read(FSDataInputStream.java:89) org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1223) org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockDataInternal(HFileBlock.java:1430) org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockData(HFileBlock.java:1312) org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:387) org.apache.hadoop.hbase.util.CompoundBloomFilter.contains(CompoundBloomFilter.java:99) org.apache.hadoop.hbase.regionserver.StoreFile$Reader.passesGeneralBloomFilter(StoreFile.java:1283) org.apache.hadoop.hbase.regionserver.StoreFile$Reader.passesBloomFilter(StoreFile.java:1148) org.apache.hadoop.hbase.regionserver.StoreFileScanner.shouldUseScanner(StoreFileScanner.java:403) org.apache.hadoop.hbase.regionserver.StoreScanner.selectScannersFrom(StoreScanner.java:363) org.apache.hadoop.hbase.regionserver.StoreScanner.getScannersNoCompaction(StoreScanner.java:291) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:170) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) Thread 109 (RW.default.writeRpcServer.handler=27,queue=7,port=60020): State: WAITING Blocked count: 15109704 Waited count: 15427560 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 108 (RW.default.writeRpcServer.handler=26,queue=6,port=60020): State: WAITING Blocked count: 15000334 Waited count: 15320876 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 107 (RW.default.writeRpcServer.handler=25,queue=5,port=60020): State: WAITING Blocked count: 15005773 Waited count: 15327385 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@41b0bc07 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 106 (RW.default.writeRpcServer.handler=24,queue=4,port=60020): State: WAITING Blocked count: 14985053 Waited count: 15306773 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 105 (RW.default.writeRpcServer.handler=23,queue=3,port=60020): State: RUNNABLE Blocked count: 15040878 Waited count: 15361917 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 104 (RW.default.writeRpcServer.handler=22,queue=2,port=60020): State: WAITING Blocked count: 15050608 Waited count: 15371739 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 103 (RW.default.writeRpcServer.handler=21,queue=1,port=60020): State: BLOCKED Blocked count: 14984041 Waited count: 15303823 Blocked on org.apache.hadoop.conf.Configuration@5a5e3da Blocked by 197 (RW.default.readRpcServer.handler=114,queue=15,port=60020) Stack: org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2250) org.apache.hadoop.conf.Configuration.get(Configuration.java:861) org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:880) org.apache.hadoop.conf.Configuration.getBoolean(Configuration.java:1281) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:138) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:157) org.apache.hadoop.hbase.regionserver.HStore.createScanner(HStore.java:1804) org.apache.hadoop.hbase.regionserver.HStore.getScanner(HStore.java:1794) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3852) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1952) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1938) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1915) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4872) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5408) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) Thread 102 (RW.default.writeRpcServer.handler=20,queue=0,port=60020): State: RUNNABLE Blocked count: 15014435 Waited count: 15335652 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 101 (RW.default.writeRpcServer.handler=19,queue=9,port=60020): State: WAITING Blocked count: 16748197 Waited count: 17085598 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 100 (RW.default.writeRpcServer.handler=18,queue=8,port=60020): State: WAITING Blocked count: 15024693 Waited count: 15345311 Waiting on java.util.concurrent.atomic.AtomicLong@71a486e9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 99 (RW.default.writeRpcServer.handler=17,queue=7,port=60020): State: WAITING Blocked count: 15025830 Waited count: 15347497 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 98 (RW.default.writeRpcServer.handler=16,queue=6,port=60020): State: WAITING Blocked count: 15046217 Waited count: 15367035 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 97 (RW.default.writeRpcServer.handler=15,queue=5,port=60020): State: RUNNABLE Blocked count: 15030993 Waited count: 15351290 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 96 (RW.default.writeRpcServer.handler=14,queue=4,port=60020): State: WAITING Blocked count: 15048883 Waited count: 15368720 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3a089142 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 95 (RW.default.writeRpcServer.handler=13,queue=3,port=60020): State: RUNNABLE Blocked count: 15047705 Waited count: 15367791 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 94 (RW.default.writeRpcServer.handler=12,queue=2,port=60020): State: WAITING Blocked count: 14971648 Waited count: 15293430 Waiting on java.lang.Object@7b2d07ac Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 93 (RW.default.writeRpcServer.handler=11,queue=1,port=60020): State: WAITING Blocked count: 15011372 Waited count: 15331639 Waiting on java.lang.Object@7d4648f7 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 92 (RW.default.writeRpcServer.handler=10,queue=0,port=60020): State: WAITING Blocked count: 15015360 Waited count: 15335736 Waiting on java.lang.Object@68956548 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 91 (RW.default.writeRpcServer.handler=9,queue=9,port=60020): State: WAITING Blocked count: 16746003 Waited count: 17083499 Waiting on java.lang.Object@46419b79 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 90 (RW.default.writeRpcServer.handler=8,queue=8,port=60020): State: WAITING Blocked count: 14977488 Waited count: 15300550 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@414a76ff Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 89 (RW.default.writeRpcServer.handler=7,queue=7,port=60020): State: WAITING Blocked count: 15066893 Waited count: 15385507 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 88 (RW.default.writeRpcServer.handler=6,queue=6,port=60020): State: WAITING Blocked count: 15044460 Waited count: 15365187 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47d0617e Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:113) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 87 (RW.default.writeRpcServer.handler=5,queue=5,port=60020): State: BLOCKED Blocked count: 15097748 Waited count: 15416159 Blocked on java.util.concurrent.atomic.AtomicLong@71a486e9 Blocked by 172 (RW.default.writeRpcServer.handler=89,queue=9,port=60020) Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 86 (RW.default.writeRpcServer.handler=4,queue=4,port=60020): State: WAITING Blocked count: 15041910 Waited count: 15364835 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5523) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 85 (RW.default.writeRpcServer.handler=3,queue=3,port=60020): State: BLOCKED Blocked count: 15019249 Waited count: 15339149 Blocked on java.util.concurrent.atomic.AtomicLong@71a486e9 Blocked by 172 (RW.default.writeRpcServer.handler=89,queue=9,port=60020) Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog.syncer(FSHLog.java:1371) org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1485) org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:6045) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5519) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 84 (RW.default.writeRpcServer.handler=2,queue=2,port=60020): State: RUNNABLE Blocked count: 14952524 Waited count: 15277374 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 83 (RW.default.writeRpcServer.handler=1,queue=1,port=60020): State: BLOCKED Blocked count: 15037013 Waited count: 15356630 Blocked on java.lang.Object@7d4648f7 Blocked by 175 (RW.default.writeRpcServer.handler=92,queue=2,port=60020) Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 82 (RW.default.writeRpcServer.handler=0,queue=0,port=60020): State: WAITING Blocked count: 15037967 Waited count: 15359946 Waiting on java.lang.Object@3c2d58cf Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForRead(MultiVersionConsistencyControl.java:152) org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.completeMemstoreInsert(MultiVersionConsistencyControl.java:87) org.apache.hadoop.hbase.regionserver.HRegion.increment(HRegion.java:5379) org.apache.hadoop.hbase.regionserver.HRegionServer.increment(HRegionServer.java:4318) org.apache.hadoop.hbase.regionserver.HRegionServer.doNonAtomicRegionMutation(HRegionServer.java:3563) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3496) org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:29931) org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2031) org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:116) org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:96) java.lang.Thread.run(Thread.java:745) Thread 16 (RpcServer.listener,port=60020): State: RUNNABLE Blocked count: 7327 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener.run(RpcServer.java:688) Thread 28 (RpcServer.responder): State: RUNNABLE Blocked count: 386 Waited count: 318 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:878) org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:861) Thread 81 (hbase-left-slave3:60020Replication Statistics #0): State: TIMED_WAITING Blocked count: 0 Waited count: 239 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 79 (regionserver60020-EventThread): State: WAITING Blocked count: 0 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@1cd21798 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Thread 78 (regionserver60020-SendThread(hbase-left-master1.hyuga.local:2181)): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Thread 76 (regionserver60020.replicationSource,1): State: RUNNABLE Blocked count: 4799268 Waited count: 7659875 Stack: sun.nio.ch.FileDispatcherImpl.write0(Native Method) sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) sun.nio.ch.IOUtil.write(IOUtil.java:65) sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:487) org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:63) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159) org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) java.io.DataOutputStream.flush(DataOutputStream.java:123) org.apache.hadoop.hdfs.protocol.datatransfer.Sender.send(Sender.java:78) org.apache.hadoop.hdfs.protocol.datatransfer.Sender.readBlock(Sender.java:109) org.apache.hadoop.hdfs.RemoteBlockReader2.newBlockReader(RemoteBlockReader2.java:400) org.apache.hadoop.hdfs.BlockReaderFactory.getRemoteBlockReader(BlockReaderFactory.java:785) org.apache.hadoop.hdfs.BlockReaderFactory.getRemoteBlockReaderFromTcp(BlockReaderFactory.java:663) org.apache.hadoop.hdfs.BlockReaderFactory.build(BlockReaderFactory.java:327) org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:574) org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:797) Thread 51 (regionserver60020.leaseChecker): State: TIMED_WAITING Blocked count: 0 Waited count: 20311 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:95) java.lang.Thread.run(Thread.java:745) Thread 53 (regionserver60020.nonceCleaner): State: TIMED_WAITING Blocked count: 0 Waited count: 218 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) org.apache.hadoop.hbase.Chore.run(Chore.java:95) java.lang.Thread.run(Thread.java:745) Thread 50 (regionserver60020.periodicFlusher): State: TIMED_WAITING Blocked count: 0 Waited count: 7142 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) org.apache.hadoop.hbase.Chore.run(Chore.java:95) java.lang.Thread.run(Thread.java:745) Thread 49 (regionserver60020.compactionChecker): State: TIMED_WAITING Blocked count: 0 Waited count: 7142 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) org.apache.hadoop.hbase.Chore.run(Chore.java:95) java.lang.Thread.run(Thread.java:745) Thread 74 (MemStoreFlusher.1): State: TIMED_WAITING Blocked count: 3190 Waited count: 12705 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.DelayQueue.poll(DelayQueue.java:256) java.util.concurrent.DelayQueue.poll(DelayQueue.java:68) org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:233) java.lang.Thread.run(Thread.java:745) Thread 72 (MemStoreFlusher.0): State: TIMED_WAITING Blocked count: 2949 Waited count: 12324 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.DelayQueue.poll(DelayQueue.java:256) java.util.concurrent.DelayQueue.poll(DelayQueue.java:68) org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:233) java.lang.Thread.run(Thread.java:745) Thread 60 (regionserver60020.logRoller): State: TIMED_WAITING Blocked count: 13261 Waited count: 25451 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:78) java.lang.Thread.run(Thread.java:745) Thread 71 (HBase-Metrics2-1): State: TIMED_WAITING Blocked count: 1 Waited count: 23801 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 70 (regionserver60020-WAL.AsyncNotifier): State: WAITING Blocked count: 93987299 Waited count: 93875926 Waiting on java.lang.Object@61ad51b9 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncNotifier.run(FSHLog.java:1338) java.lang.Thread.run(Thread.java:745) Thread 69 (regionserver60020-WAL.AsyncSyncer4): State: WAITING Blocked count: 98413795 Waited count: 97949005 Waiting on java.lang.Object@39d07ac8 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncSyncer.run(FSHLog.java:1209) java.lang.Thread.run(Thread.java:745) Thread 68 (regionserver60020-WAL.AsyncSyncer3): State: WAITING Blocked count: 116827334 Waited count: 116317789 Waiting on java.lang.Object@262cfe1f Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncSyncer.run(FSHLog.java:1209) java.lang.Thread.run(Thread.java:745) Thread 67 (regionserver60020-WAL.AsyncSyncer2): State: WAITING Blocked count: 134111753 Waited count: 133564534 Waiting on java.lang.Object@53dfb8cd Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncSyncer.run(FSHLog.java:1209) java.lang.Thread.run(Thread.java:745) Thread 66 (regionserver60020-WAL.AsyncSyncer1): State: WAITING Blocked count: 149307450 Waited count: 148737264 Waiting on java.lang.Object@4ddc5c0 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncSyncer.run(FSHLog.java:1209) java.lang.Thread.run(Thread.java:745) Thread 65 (regionserver60020-WAL.AsyncSyncer0): State: RUNNABLE Blocked count: 158858834 Waited count: 158272502 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:2061) org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1951) org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1859) org.apache.hadoop.hdfs.DFSOutputStream.sync(DFSOutputStream.java:1844) org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:123) org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.sync(ProtobufLogWriter.java:165) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncSyncer.run(FSHLog.java:1250) java.lang.Thread.run(Thread.java:745) Thread 64 (regionserver60020-WAL.AsyncWriter): State: RUNNABLE Blocked count: 151135116 Waited count: 147911738 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) org.apache.hadoop.hbase.regionserver.wal.FSHLog$AsyncWriter.run(FSHLog.java:1116) java.lang.Thread.run(Thread.java:745) Thread 62 (LeaseRenewer:hbase@left): State: TIMED_WAITING Blocked count: 2368 Waited count: 76108 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438) org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71) org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298) java.lang.Thread.run(Thread.java:745) Thread 59 (regionserver60020-EventThread): State: WAITING Blocked count: 0 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4a2d12b Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Thread 58 (regionserver60020-SendThread(hbase-right-master2.hyuga.local:2181)): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Thread 54 (JvmPauseMonitor): State: TIMED_WAITING Blocked count: 0 Waited count: 142653 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:159) java.lang.Thread.run(Thread.java:745) Thread 44 (Thread-10): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) org.apache.hadoop.net.unix.DomainSocketWatcher$1.run(DomainSocketWatcher.java:474) java.lang.Thread.run(Thread.java:745) Thread 43 (regionserver60020-EventThread): State: WAITING Blocked count: 0 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@80baa82 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Thread 42 (regionserver60020-SendThread(hbase-left-master1.hyuga.local:2181)): State: RUNNABLE Blocked count: 3 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Thread 40 (regionserver60020-EventThread): State: WAITING Blocked count: 36 Waited count: 47 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@35e740ee Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Thread 39 (regionserver60020-SendThread(hbase-left-master3.hyuga.local:2181)): State: RUNNABLE Blocked count: 36033 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Thread 38 (regionserver60020): State: TIMED_WAITING Blocked count: 23883 Waited count: 47648 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:56) org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:935) java.lang.Thread.run(Thread.java:745) Thread 37 (Timer-0): State: TIMED_WAITING Blocked count: 0 Waited count: 2381 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Thread 36 (639678835@qtp-1360685958-1 - Acceptor0 HttpServer$SelectChannelConnectorWithSafeStartup@0.0.0.0:60030): State: RUNNABLE Blocked count: 1285 Waited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:511) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:193) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Thread 35 (109889466@qtp-1360685958-0): State: RUNNABLE Blocked count: 10443 Waited count: 10483 Stack: sun.management.ThreadImpl.getThreadInfo1(Native Method) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:174) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:139) org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:165) org.apache.hadoop.hbase.regionserver.RSDumpServlet.doGet(RSDumpServlet.java:81) javax.servlet.http.HttpServlet.service(HttpServlet.java:707) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:1122) org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) org.apache.hadoop.http.NoCacheFilter.doFilter(NoCacheFilter.java:45) org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767) org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) Thread 33 (MobFileCache #0): State: TIMED_WAITING Blocked count: 0 Waited count: 20 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 31 (LruStats #0): State: TIMED_WAITING Blocked count: 0 Waited count: 239 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 29 (main.LruBlockCache.EvictionThread): State: TIMED_WAITING Blocked count: 972 Waited count: 7297 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:705) java.lang.Thread.run(Thread.java:745) Thread 27 (Timer for 'HBase' metrics system): State: TIMED_WAITING Blocked count: 0 Waited count: 7143 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Thread 26 (RpcServer.reader=9,port=60020): State: RUNNABLE Blocked count: 31531 Waited count: 207331 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 25 (RpcServer.reader=8,port=60020): State: RUNNABLE Blocked count: 31171 Waited count: 201339 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 24 (RpcServer.reader=7,port=60020): State: RUNNABLE Blocked count: 31556 Waited count: 207604 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 23 (RpcServer.reader=6,port=60020): State: RUNNABLE Blocked count: 31636 Waited count: 207348 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 22 (RpcServer.reader=5,port=60020): State: RUNNABLE Blocked count: 31503 Waited count: 209926 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 21 (RpcServer.reader=4,port=60020): State: RUNNABLE Blocked count: 31390 Waited count: 205851 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 20 (RpcServer.reader=3,port=60020): State: RUNNABLE Blocked count: 31578 Waited count: 206474 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 19 (RpcServer.reader=2,port=60020): State: RUNNABLE Blocked count: 31041 Waited count: 202746 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 18 (RpcServer.reader=1,port=60020): State: RUNNABLE Blocked count: 31720 Waited count: 213236 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 17 (RpcServer.reader=0,port=60020): State: RUNNABLE Blocked count: 31268 Waited count: 210374 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:582) org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:568) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 13 (event filterer): State: TIMED_WAITING Blocked count: 4 Waited count: 57980 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.runFiltering(LogEventProcessor.java:132) com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.access$000(LogEventProcessor.java:28) com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor$1.run(LogEventProcessor.java:81) Thread 12 (RMI TCP Accept-0): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: java.net.PlainSocketImpl.socketAccept(Native Method) java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) java.net.ServerSocket.implAccept(ServerSocket.java:530) java.net.ServerSocket.accept(ServerSocket.java:498) sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:388) sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:360) java.lang.Thread.run(Thread.java:745) Thread 11 (RMI TCP Accept-10102): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: java.net.PlainSocketImpl.socketAccept(Native Method) java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) java.net.ServerSocket.implAccept(ServerSocket.java:530) java.net.ServerSocket.accept(ServerSocket.java:498) sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:388) sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:360) java.lang.Thread.run(Thread.java:745) Thread 10 (RMI TCP Accept-0): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: java.net.PlainSocketImpl.socketAccept(Native Method) java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) java.net.ServerSocket.implAccept(ServerSocket.java:530) java.net.ServerSocket.accept(ServerSocket.java:498) sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:388) sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:360) java.lang.Thread.run(Thread.java:745) Thread 5 (Signal Dispatcher): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: Thread 3 (Finalizer): State: WAITING Blocked count: 89424 Waited count: 54726 Waiting on java.lang.ref.ReferenceQueue$Lock@634a8d33 Stack: java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Thread 2 (Reference Handler): State: WAITING Blocked count: 58605 Waited count: 54117 Waiting on java.lang.ref.Reference$Lock@337c4232 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) Thread 1 (main): State: WAITING Blocked count: 41 Waited count: 2 Waiting on java.lang.Thread@2356df1d Stack: java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1281) java.lang.Thread.join(Thread.java:1355) org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:64) org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:85) org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126) org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2472)