2011-07-14 12:30:07 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x00002aad49576800 nid=0x7707 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ResponseProcessor for block blk_3448678095430183883_297684" daemon prio=10 tid=0x0000000048307800 nid=0x7305 runnable [0x0000000043af9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00002aabd22362e8> (a sun.nio.ch.Util$1) - locked <0x00002aabd22362d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00002aabc59faf48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:178) at java.io.DataInputStream.readLong(DataInputStream.java:399) at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2622) "DataStreamer for file /hbase/.logs/H1HS3,60020,1310505473349/H1HS3%3A60020.1310665634920 block blk_3448678095430183883_297684" daemon prio=10 tid=0x00002aad5802d000 nid=0x7302 in Object.wait() [0x000000004d28d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab77d0a60> (a java.util.LinkedList) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2467) - locked <0x00002aaab77d0a60> (a java.util.LinkedList) "IPC Client (47) connection to H1HM1/10.5.240.233:60000 from an unknown user" daemon prio=10 tid=0x00002aad5403c000 nid=0x1189 in Object.wait() [0x0000000040b52000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aabda747448> (a org.apache.hadoop.hbase.ipc.HBaseClient$Connection) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:431) - locked <0x00002aabda747448> (a org.apache.hadoop.hbase.ipc.HBaseClient$Connection) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:476) "RS_CLOSE_REGION-H1HS3,60020,1310505473349-2" daemon prio=10 tid=0x00000000482b1000 nid=0x8f4 waiting on condition [0x000000004d18c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8894ba0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_CLOSE_REGION-H1HS3,60020,1310505473349-1" daemon prio=10 tid=0x0000000047cb7800 nid=0x8f3 waiting on condition [0x000000004040d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8894ba0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_CLOSE_REGION-H1HS3,60020,1310505473349-0" daemon prio=10 tid=0x00000000484ff000 nid=0x8f2 waiting on condition [0x000000004d08b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8894ba0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-H1HS3,60020,1310505473349-2" daemon prio=10 tid=0x0000000047be8800 nid=0x126b waiting on condition [0x000000004cf8a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8894d80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-H1HS3,60020,1310505473349-1" daemon prio=10 tid=0x0000000048226800 nid=0x126a waiting on condition [0x000000004ce89000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8894d80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-H1HS3,60020,1310505473349-0" daemon prio=10 tid=0x0000000048225800 nid=0x1269 waiting on condition [0x000000004cd88000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8894d80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LRU Statistics #0" prio=10 tid=0x00002aad446ca800 nid=0x125a waiting on condition [0x000000004ca85000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab887f2b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LruBlockCache.EvictionThread" daemon prio=10 tid=0x00002aad446cc800 nid=0x1259 in Object.wait() [0x000000004c984000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab9f65cf8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:519) - locked <0x00002aaab9f65cf8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) "PRI IPC Server handler 9 on 60020" daemon prio=10 tid=0x00002aad446c7800 nid=0x1258 waiting on condition [0x000000004c883000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 8 on 60020" daemon prio=10 tid=0x00002aad446c5800 nid=0x1257 waiting on condition [0x000000004c782000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 7 on 60020" daemon prio=10 tid=0x00002aad446c3800 nid=0x1256 waiting on condition [0x000000004c681000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 6 on 60020" daemon prio=10 tid=0x00002aad446c1800 nid=0x1255 waiting on condition [0x000000004c580000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 5 on 60020" daemon prio=10 tid=0x00002aad446bf800 nid=0x1254 waiting on condition [0x000000004c47f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 4 on 60020" daemon prio=10 tid=0x00002aad446bd800 nid=0x1253 waiting on condition [0x000000004c37e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 3 on 60020" daemon prio=10 tid=0x00002aad446bc000 nid=0x1252 waiting on condition [0x000000004c27d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 2 on 60020" daemon prio=10 tid=0x00002aad44812800 nid=0x1251 waiting on condition [0x000000004c17c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 1 on 60020" daemon prio=10 tid=0x00002aad44810800 nid=0x1250 waiting on condition [0x000000004c07b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 0 on 60020" daemon prio=10 tid=0x00002aad4480e800 nid=0x124f waiting on condition [0x000000004bf7a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1bd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "IPC Server handler 99 on 60020" daemon prio=10 tid=0x00002aad4480c800 nid=0x124e waiting for monitor entry [0x000000004be79000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 98 on 60020" daemon prio=10 tid=0x00002aad4480a800 nid=0x124d waiting for monitor entry [0x000000004bd78000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 97 on 60020" daemon prio=10 tid=0x00002aad44808800 nid=0x124c waiting for monitor entry [0x000000004bc77000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 96 on 60020" daemon prio=10 tid=0x00002aad44807000 nid=0x124b waiting for monitor entry [0x000000004bb76000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 95 on 60020" daemon prio=10 tid=0x00002aad44805000 nid=0x124a waiting for monitor entry [0x000000004ba75000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 94 on 60020" daemon prio=10 tid=0x00002aad44803000 nid=0x1249 waiting for monitor entry [0x000000004b974000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 93 on 60020" daemon prio=10 tid=0x00002aad44801000 nid=0x1248 waiting for monitor entry [0x000000004b873000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 92 on 60020" daemon prio=10 tid=0x00002aad447ff000 nid=0x1247 waiting for monitor entry [0x000000004b772000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 91 on 60020" daemon prio=10 tid=0x00002aad447fd000 nid=0x1246 waiting for monitor entry [0x000000004b671000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 90 on 60020" daemon prio=10 tid=0x00002aad447fb000 nid=0x1245 waiting for monitor entry [0x000000004b570000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 89 on 60020" daemon prio=10 tid=0x00002aad447f9000 nid=0x1244 waiting for monitor entry [0x000000004b46f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 88 on 60020" daemon prio=10 tid=0x00002aad447f7000 nid=0x1243 waiting for monitor entry [0x000000004b36e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 87 on 60020" daemon prio=10 tid=0x00002aad447f5000 nid=0x1242 waiting for monitor entry [0x000000004b26d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 86 on 60020" daemon prio=10 tid=0x00002aad447f3000 nid=0x1241 waiting for monitor entry [0x000000004b16c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 85 on 60020" daemon prio=10 tid=0x00002aad447f1800 nid=0x1240 waiting for monitor entry [0x000000004b06b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 84 on 60020" daemon prio=10 tid=0x00002aad447ef800 nid=0x123f waiting for monitor entry [0x000000004af6a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 83 on 60020" daemon prio=10 tid=0x00002aad447ed800 nid=0x123e waiting for monitor entry [0x000000004ae69000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 82 on 60020" daemon prio=10 tid=0x00002aad447eb800 nid=0x123d waiting for monitor entry [0x000000004ad68000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 81 on 60020" daemon prio=10 tid=0x00002aad447e9800 nid=0x123c waiting for monitor entry [0x000000004ac67000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 80 on 60020" daemon prio=10 tid=0x00002aad447e7800 nid=0x123b waiting for monitor entry [0x000000004ab66000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 79 on 60020" daemon prio=10 tid=0x00002aad447e5800 nid=0x123a waiting for monitor entry [0x000000004aa65000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 78 on 60020" daemon prio=10 tid=0x00002aad447e3800 nid=0x1239 waiting for monitor entry [0x000000004a964000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 77 on 60020" daemon prio=10 tid=0x00002aad447e1800 nid=0x1238 waiting for monitor entry [0x000000004a863000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 76 on 60020" daemon prio=10 tid=0x00002aad447df800 nid=0x1237 waiting for monitor entry [0x000000004a762000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 75 on 60020" daemon prio=10 tid=0x00002aad447dd800 nid=0x1236 waiting for monitor entry [0x000000004a661000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 74 on 60020" daemon prio=10 tid=0x00002aad447db800 nid=0x1235 waiting for monitor entry [0x000000004a560000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 73 on 60020" daemon prio=10 tid=0x00002aad447da000 nid=0x1234 waiting for monitor entry [0x000000004a45f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 72 on 60020" daemon prio=10 tid=0x00002aad447d8000 nid=0x1233 waiting for monitor entry [0x000000004a35e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 71 on 60020" daemon prio=10 tid=0x00002aad447d6800 nid=0x1232 waiting for monitor entry [0x000000004a25d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 70 on 60020" daemon prio=10 tid=0x00002aad447d3000 nid=0x1231 waiting for monitor entry [0x000000004a15c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 69 on 60020" daemon prio=10 tid=0x00002aad447d1000 nid=0x1230 waiting for monitor entry [0x000000004a05b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 68 on 60020" daemon prio=10 tid=0x00002aad447cf000 nid=0x122f waiting for monitor entry [0x0000000049f5a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 67 on 60020" daemon prio=10 tid=0x00002aad447cd000 nid=0x122e waiting for monitor entry [0x0000000049e59000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 66 on 60020" daemon prio=10 tid=0x00002aad447cb000 nid=0x122d waiting for monitor entry [0x0000000049d58000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 65 on 60020" daemon prio=10 tid=0x00002aad447c9000 nid=0x122c waiting for monitor entry [0x0000000049c57000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 64 on 60020" daemon prio=10 tid=0x00002aad447c7000 nid=0x122b waiting on condition [0x0000000049b56000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8eea130> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:747) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:778) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1114) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:435) - locked <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 63 on 60020" daemon prio=10 tid=0x00002aad447c5800 nid=0x122a waiting for monitor entry [0x0000000049a55000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 62 on 60020" daemon prio=10 tid=0x00002aad447c3800 nid=0x1229 waiting for monitor entry [0x0000000049954000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 61 on 60020" daemon prio=10 tid=0x00002aad447c1800 nid=0x1228 waiting for monitor entry [0x0000000049853000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 60 on 60020" daemon prio=10 tid=0x00002aad447bf800 nid=0x1227 waiting for monitor entry [0x0000000049752000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 59 on 60020" daemon prio=10 tid=0x00002aad447bd800 nid=0x1226 waiting for monitor entry [0x0000000049651000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 58 on 60020" daemon prio=10 tid=0x00002aad447bb800 nid=0x1225 waiting for monitor entry [0x0000000049550000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 57 on 60020" daemon prio=10 tid=0x00002aad447b9800 nid=0x1224 waiting for monitor entry [0x000000004944f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 56 on 60020" daemon prio=10 tid=0x00002aad447b7800 nid=0x1223 waiting for monitor entry [0x000000004934e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 55 on 60020" daemon prio=10 tid=0x00002aad447b5800 nid=0x1222 waiting for monitor entry [0x000000004924d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 54 on 60020" daemon prio=10 tid=0x00002aad447b3800 nid=0x1221 waiting for monitor entry [0x000000004914c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 53 on 60020" daemon prio=10 tid=0x00002aad447b1800 nid=0x1220 waiting for monitor entry [0x000000004904b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 52 on 60020" daemon prio=10 tid=0x00002aad447b0000 nid=0x121f waiting for monitor entry [0x0000000048f4a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 51 on 60020" daemon prio=10 tid=0x00002aad4478b000 nid=0x121e waiting for monitor entry [0x0000000048e49000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 50 on 60020" daemon prio=10 tid=0x00002aad44789000 nid=0x121d waiting for monitor entry [0x0000000048d48000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 49 on 60020" daemon prio=10 tid=0x00002aad44787000 nid=0x121c waiting for monitor entry [0x0000000047634000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 48 on 60020" daemon prio=10 tid=0x00002aad44785000 nid=0x121b waiting for monitor entry [0x0000000047533000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 47 on 60020" daemon prio=10 tid=0x00002aad44783000 nid=0x121a waiting for monitor entry [0x0000000047432000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 46 on 60020" daemon prio=10 tid=0x00002aad44781000 nid=0x1219 waiting for monitor entry [0x0000000047331000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 45 on 60020" daemon prio=10 tid=0x00002aad4477f000 nid=0x1218 waiting for monitor entry [0x0000000047230000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 44 on 60020" daemon prio=10 tid=0x00002aad4477d000 nid=0x1217 waiting for monitor entry [0x000000004712f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 43 on 60020" daemon prio=10 tid=0x00002aad4477b800 nid=0x1216 waiting for monitor entry [0x000000004702e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 42 on 60020" daemon prio=10 tid=0x00002aad44779800 nid=0x1215 waiting for monitor entry [0x0000000046f2d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 41 on 60020" daemon prio=10 tid=0x00002aad44777800 nid=0x1214 waiting for monitor entry [0x0000000046e2c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 40 on 60020" daemon prio=10 tid=0x00002aad44775800 nid=0x1213 waiting for monitor entry [0x0000000046d2b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 39 on 60020" daemon prio=10 tid=0x00002aad44773800 nid=0x1212 waiting for monitor entry [0x0000000046c2a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 38 on 60020" daemon prio=10 tid=0x00002aad44771800 nid=0x1211 waiting for monitor entry [0x0000000046b29000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 37 on 60020" daemon prio=10 tid=0x00002aad4476f800 nid=0x1210 waiting for monitor entry [0x0000000046a28000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 36 on 60020" daemon prio=10 tid=0x00002aad4476d800 nid=0x120f waiting for monitor entry [0x0000000046927000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 35 on 60020" daemon prio=10 tid=0x00002aad4476c000 nid=0x120e waiting for monitor entry [0x0000000046826000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 34 on 60020" daemon prio=10 tid=0x00002aad44759800 nid=0x120d waiting for monitor entry [0x0000000046725000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 33 on 60020" daemon prio=10 tid=0x00002aad44757800 nid=0x120c waiting for monitor entry [0x0000000046624000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 32 on 60020" daemon prio=10 tid=0x00002aad44755800 nid=0x120b waiting for monitor entry [0x0000000046523000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 31 on 60020" daemon prio=10 tid=0x00002aad44754000 nid=0x120a waiting for monitor entry [0x0000000046422000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 30 on 60020" daemon prio=10 tid=0x00002aad44752000 nid=0x1209 waiting for monitor entry [0x0000000046321000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 29 on 60020" daemon prio=10 tid=0x00002aad44750000 nid=0x1208 waiting for monitor entry [0x0000000046220000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 28 on 60020" daemon prio=10 tid=0x00002aad4474e000 nid=0x1207 waiting for monitor entry [0x000000004611f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 27 on 60020" daemon prio=10 tid=0x00002aad4474c000 nid=0x1206 waiting for monitor entry [0x000000004601e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 26 on 60020" daemon prio=10 tid=0x00002aad4474a000 nid=0x1205 waiting for monitor entry [0x0000000045f1d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 25 on 60020" daemon prio=10 tid=0x00002aad44748000 nid=0x1204 waiting for monitor entry [0x0000000045e1c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 24 on 60020" daemon prio=10 tid=0x00002aad44746000 nid=0x1203 waiting for monitor entry [0x0000000045d1b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 23 on 60020" daemon prio=10 tid=0x00002aad446ae000 nid=0x1202 waiting for monitor entry [0x0000000045c1a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 22 on 60020" daemon prio=10 tid=0x00002aad446ac000 nid=0x1201 waiting for monitor entry [0x0000000045b19000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 21 on 60020" daemon prio=10 tid=0x00002aad446aa800 nid=0x1200 waiting for monitor entry [0x0000000045a18000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 20 on 60020" daemon prio=10 tid=0x00002aad446a8800 nid=0x11ff waiting for monitor entry [0x0000000045917000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 19 on 60020" daemon prio=10 tid=0x00002aad446a6800 nid=0x11fe waiting for monitor entry [0x0000000045816000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 18 on 60020" daemon prio=10 tid=0x00002aad446a4800 nid=0x11fd waiting for monitor entry [0x0000000045715000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 17 on 60020" daemon prio=10 tid=0x00002aad446a2800 nid=0x11fc waiting for monitor entry [0x0000000045614000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 16 on 60020" daemon prio=10 tid=0x00002aad446a0800 nid=0x11fb waiting for monitor entry [0x0000000045513000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 15 on 60020" daemon prio=10 tid=0x00002aad4479e800 nid=0x11fa waiting for monitor entry [0x0000000045412000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 14 on 60020" daemon prio=10 tid=0x00002aad4479c800 nid=0x11f9 waiting for monitor entry [0x0000000045311000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 13 on 60020" daemon prio=10 tid=0x00002aad4479a800 nid=0x11f8 waiting for monitor entry [0x0000000045210000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 12 on 60020" daemon prio=10 tid=0x00002aad44798800 nid=0x11f7 waiting for monitor entry [0x000000004510f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 11 on 60020" daemon prio=10 tid=0x00002aad44796800 nid=0x11f6 waiting for monitor entry [0x000000004500e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 10 on 60020" daemon prio=10 tid=0x00002aad4466e800 nid=0x11f5 waiting for monitor entry [0x0000000044f0d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 9 on 60020" daemon prio=10 tid=0x00002aad4466c800 nid=0x11f4 waiting for monitor entry [0x0000000044e0c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 8 on 60020" daemon prio=10 tid=0x00002aad4466a800 nid=0x11f3 waiting for monitor entry [0x0000000044d0b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 7 on 60020" daemon prio=10 tid=0x00002aad44668800 nid=0x11f2 waiting for monitor entry [0x0000000044c0a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 6 on 60020" daemon prio=10 tid=0x00002aad44666800 nid=0x11f1 waiting for monitor entry [0x0000000044b09000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 5 on 60020" daemon prio=10 tid=0x00002aad44694800 nid=0x11f0 waiting for monitor entry [0x0000000044a08000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 4 on 60020" daemon prio=10 tid=0x00002aad44692800 nid=0x11ef waiting for monitor entry [0x0000000044907000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 3 on 60020" daemon prio=10 tid=0x00002aad44690800 nid=0x11ee waiting for monitor entry [0x0000000044806000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 2 on 60020" daemon prio=10 tid=0x00002aad4468f000 nid=0x11ed waiting for monitor entry [0x0000000044705000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 1 on 60020" daemon prio=10 tid=0x00002aad4456f000 nid=0x11ec waiting for monitor entry [0x0000000044604000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 0 on 60020" daemon prio=10 tid=0x00002aad4456e800 nid=0x11eb waiting for monitor entry [0x0000000044503000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server listener on 60020" daemon prio=10 tid=0x00002aad44710800 nid=0x11ea waiting for monitor entry [0x0000000044402000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.registerChannel(HBaseServer.java:347) - waiting to lock <0x00002aaab885ba68> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doAccept(HBaseServer.java:496) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:422) "IPC Server Responder" daemon prio=10 tid=0x00002aad44674800 nid=0x11e9 runnable [0x0000000044301000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00002aaab89b10c0> (a sun.nio.ch.Util$1) - locked <0x00002aaab89b10a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00002aaab88c4430> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:588) "Timer-0" daemon prio=10 tid=0x0000000047e5e000 nid=0x11e8 in Object.wait() [0x0000000040950000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab8974ec0> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x00002aaab8974ec0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "1809605797@qtp-780857040-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030" prio=10 tid=0x0000000047f9a800 nid=0x11e7 runnable [0x0000000044200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00002aaaba2b76f0> (a sun.nio.ch.Util$1) - locked <0x00002aaaba2b7708> (a java.util.Collections$UnmodifiableSet) - locked <0x00002aaab88b7440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1604848362@qtp-780857040-0" prio=10 tid=0x00000000481e2800 nid=0x11e6 in Object.wait() [0x00000000440ff000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab8f005a0> (a org.mortbay.thread.QueuedThreadPool$PoolThread) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x00002aaab8f005a0> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "regionserver60020.leaseChecker" daemon prio=10 tid=0x00002aad44563000 nid=0x11e1 waiting on condition [0x0000000043ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8f2e7c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:201) at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:82) "regionserver60020.majorCompactionChecker" daemon prio=10 tid=0x00002aad44557000 nid=0x11e0 in Object.wait() [0x0000000041885000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab8f28070> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) - locked <0x00002aaab8f28070> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:74) "regionserver60020.compactor" daemon prio=10 tid=0x00002aad4453e800 nid=0x11df waiting on condition [0x0000000041784000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8f2e810> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.PriorityBlockingQueue.poll(PriorityBlockingQueue.java:245) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.poll(PriorityCompactionQueue.java:248) at org.apache.hadoop.hbase.regionserver.CompactSplitThread.run(CompactSplitThread.java:75) "regionserver60020.cacheFlusher" daemon prio=10 tid=0x0000000047bd9000 nid=0x11de waiting for monitor entry [0x0000000040d7d000] java.lang.Thread.State: BLOCKED (on object monitor) at java.util.ResourceBundle.endLoading(ResourceBundle.java:1506) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at java.util.ResourceBundle.findBundle(ResourceBundle.java:1379) at java.util.ResourceBundle.findBundle(ResourceBundle.java:1292) at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:1234) at java.util.ResourceBundle.getBundle(ResourceBundle.java:832) at sun.util.resources.LocaleData$1.run(LocaleData.java:127) at java.security.AccessController.doPrivileged(Native Method) at sun.util.resources.LocaleData.getBundle(LocaleData.java:125) at sun.util.resources.LocaleData.getTimeZoneNames(LocaleData.java:97) at sun.util.TimeZoneNameUtility.getBundle(TimeZoneNameUtility.java:115) at sun.util.TimeZoneNameUtility.retrieveDisplayNames(TimeZoneNameUtility.java:80) at java.util.TimeZone.getDisplayNames(TimeZone.java:399) at java.util.TimeZone.getDisplayName(TimeZone.java:350) at java.util.Date.toString(Date.java:1025) at java.lang.String.valueOf(String.java:2826) at java.lang.StringBuilder.append(StringBuilder.java:115) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue$CompactionRequest.toString(PriorityCompactionQueue.java:114) at java.lang.String.valueOf(String.java:2826) at java.lang.StringBuilder.append(StringBuilder.java:115) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.addToRegionsInQueue(PriorityCompactionQueue.java:145) - locked <0x00002aaab8f2dc58> (a java.util.HashMap) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.add(PriorityCompactionQueue.java:188) at org.apache.hadoop.hbase.regionserver.CompactSplitThread.requestCompaction(CompactSplitThread.java:140) - locked <0x00002aaab8894048> (a org.apache.hadoop.hbase.regionserver.CompactSplitThread) at org.apache.hadoop.hbase.regionserver.CompactSplitThread.requestCompaction(CompactSplitThread.java:118) - locked <0x00002aaab8894048> (a org.apache.hadoop.hbase.regionserver.CompactSplitThread) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:393) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:366) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:240) "regionserver60020.logRoller" daemon prio=10 tid=0x0000000047bd7000 nid=0x11dd in Object.wait() [0x0000000040c78000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab9ead668> (a java.util.concurrent.atomic.AtomicBoolean) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:78) - locked <0x00002aaab9ead668> (a java.util.concurrent.atomic.AtomicBoolean) "Timer thread for monitoring jvm" daemon prio=10 tid=0x0000000047bd5800 nid=0x11dc in Object.wait() [0x0000000043efd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab9e38f40> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x00002aaab9e38f40> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "Timer thread for monitoring hbase" daemon prio=10 tid=0x0000000048764800 nid=0x11db in Object.wait() [0x0000000043dfc000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab9ead300> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x00002aaab9ead300> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "regionserver60020.logSyncer" daemon prio=10 tid=0x0000000048445800 nid=0x11da waiting on condition [0x0000000043cfb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:962) "LeaseChecker" daemon prio=10 tid=0x0000000048444000 nid=0x11d9 waiting on condition [0x0000000043bfa000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1151) at java.lang.Thread.run(Thread.java:619) "sendParams-0" daemon prio=10 tid=0x0000000048873800 nid=0x11d7 waiting on condition [0x00000000439f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab9e381d0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00002aad444d0000 nid=0x11ab waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "regionserver60020-EventThread" daemon prio=10 tid=0x00002aad444cb000 nid=0x11d4 waiting on condition [0x00000000438f7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8961d50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) "regionserver60020-SendThread(H1HS2:2181)" daemon prio=10 tid=0x00002aad444c9000 nid=0x11d3 runnable [0x00000000437f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00002aaab89c9318> (a sun.nio.ch.Util$1) - locked <0x00002aaab89c9330> (a java.util.Collections$UnmodifiableSet) - locked <0x00002aaab88c4610> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107) "regionserver60020" prio=10 tid=0x00002aad444ae000 nid=0x11d2 waiting on condition [0x000000004084f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:395) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:624) at java.lang.Thread.run(Thread.java:619) "Timer thread for monitoring rpc" daemon prio=10 tid=0x00002aad4445e800 nid=0x11d1 in Object.wait() [0x00000000436f5000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab8853a80> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x00002aaab8853a80> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "IPC Reader 9 on port 60020" prio=10 tid=0x00002aad4445b800 nid=0x11d0 waiting on condition [0x00000000435f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab885b9f0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 8 on port 60020" prio=10 tid=0x00002aad44440800 nid=0x11cf waiting on condition [0x00000000434f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab8852240> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 7 on port 60020" prio=10 tid=0x00002aad44327000 nid=0x11ce waiting on condition [0x00000000433f2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab8859140> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 6 on port 60020" prio=10 tid=0x00002aad44325800 nid=0x11cd waiting on condition [0x00000000432f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab885ba18> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 5 on port 60020" prio=10 tid=0x00002aad44324000 nid=0x11cc waiting on condition [0x00000000431f0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab8859168> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 4 on port 60020" prio=10 tid=0x00002aad44322000 nid=0x11cb waiting on condition [0x00000000430ef000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab885ba40> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 3 on port 60020" prio=10 tid=0x00002aad44320000 nid=0x11ca waiting on condition [0x0000000042fee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab8852268> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 2 on port 60020" prio=10 tid=0x00002aad4431e800 nid=0x11c9 waiting on condition [0x0000000042eed000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab8859190> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 1 on port 60020" prio=10 tid=0x00002aad4431c800 nid=0x11c8 waiting on condition [0x0000000042dec000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab885ba68> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "IPC Reader 0 on port 60020" prio=10 tid=0x00002aad4431b800 nid=0x11c7 waiting on condition [0x0000000042ceb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab89b1c50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:254) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x00002aaab8852290> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:619) "main-EventThread" daemon prio=10 tid=0x00002aad442ff800 nid=0x11c6 waiting on condition [0x0000000041044000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab88591b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) "main-SendThread(H1HM1:2181)" daemon prio=10 tid=0x00002aad44312000 nid=0x11c5 runnable [0x0000000040f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00002aaab8851ee8> (a sun.nio.ch.Util$1) - locked <0x00002aaab8851f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00002aaab8858830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107) "Low Memory Detector" daemon prio=10 tid=0x00002aad44002800 nid=0x11be runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00000000478ce800 nid=0x11bd waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x00000000478cc000 nid=0x11bc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000478c9800 nid=0x11bb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000478c7800 nid=0x11ba waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000478aa800 nid=0x11b9 in Object.wait() [0x00000000426e5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab8851f18> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00002aaab8851f18> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000478a3000 nid=0x11b8 in Object.wait() [0x00000000425e4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00002aaab8850018> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00002aaab8850018> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x000000004789c800 nid=0x11b7 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x00000000476db000 nid=0x11ac runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x00000000476dc800 nid=0x11ad runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x00000000476de800 nid=0x11ae runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x00000000476e0800 nid=0x11af runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x00000000476e2000 nid=0x11b0 runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x00000000476e4000 nid=0x11b1 runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x00000000476e6000 nid=0x11b2 runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x00000000476e7800 nid=0x11b3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00000000477b8000 nid=0x11b6 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x00000000477b4000 nid=0x11b4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x00000000477b6000 nid=0x11b5 runnable "VM Periodic Task Thread" prio=10 tid=0x00002aad44005000 nid=0x11bf waiting on condition JNI global references: 890 Found one Java-level deadlock: ============================= "IPC Server handler 99 on 60020": waiting to lock monitor 0x0000000047f97000 (object 0x00002aaab8ef07e8, a org.apache.hadoop.hbase.regionserver.MemStoreFlusher), which is held by "IPC Server handler 64 on 60020" "IPC Server handler 64 on 60020": waiting for ownable synchronizer 0x00002aaab8eea130, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), which is held by "regionserver60020.cacheFlusher" "regionserver60020.cacheFlusher": waiting to lock monitor 0x0000000047f97000 (object 0x00002aaab8ef07e8, a org.apache.hadoop.hbase.regionserver.MemStoreFlusher), which is held by "IPC Server handler 64 on 60020" Java stack information for the threads listed above: =================================================== "IPC Server handler 99 on 60020": at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:434) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 64 on 60020": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00002aaab8eea130> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:747) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:778) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1114) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.reclaimMemStoreMemory(MemStoreFlusher.java:435) - locked <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2529) at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "regionserver60020.cacheFlusher": at java.util.ResourceBundle.endLoading(ResourceBundle.java:1506) - waiting to lock <0x00002aaab8ef07e8> (a org.apache.hadoop.hbase.regionserver.MemStoreFlusher) at java.util.ResourceBundle.findBundle(ResourceBundle.java:1379) at java.util.ResourceBundle.findBundle(ResourceBundle.java:1292) at java.util.ResourceBundle.getBundleImpl(ResourceBundle.java:1234) at java.util.ResourceBundle.getBundle(ResourceBundle.java:832) at sun.util.resources.LocaleData$1.run(LocaleData.java:127) at java.security.AccessController.doPrivileged(Native Method) at sun.util.resources.LocaleData.getBundle(LocaleData.java:125) at sun.util.resources.LocaleData.getTimeZoneNames(LocaleData.java:97) at sun.util.TimeZoneNameUtility.getBundle(TimeZoneNameUtility.java:115) at sun.util.TimeZoneNameUtility.retrieveDisplayNames(TimeZoneNameUtility.java:80) at java.util.TimeZone.getDisplayNames(TimeZone.java:399) at java.util.TimeZone.getDisplayName(TimeZone.java:350) at java.util.Date.toString(Date.java:1025) at java.lang.String.valueOf(String.java:2826) at java.lang.StringBuilder.append(StringBuilder.java:115) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue$CompactionRequest.toString(PriorityCompactionQueue.java:114) at java.lang.String.valueOf(String.java:2826) at java.lang.StringBuilder.append(StringBuilder.java:115) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.addToRegionsInQueue(PriorityCompactionQueue.java:145) - locked <0x00002aaab8f2dc58> (a java.util.HashMap) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.add(PriorityCompactionQueue.java:188) at org.apache.hadoop.hbase.regionserver.CompactSplitThread.requestCompaction(CompactSplitThread.java:140) - locked <0x00002aaab8894048> (a org.apache.hadoop.hbase.regionserver.CompactSplitThread) at org.apache.hadoop.hbase.regionserver.CompactSplitThread.requestCompaction(CompactSplitThread.java:118) - locked <0x00002aaab8894048> (a org.apache.hadoop.hbase.regionserver.CompactSplitThread) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:393) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.flushRegion(MemStoreFlusher.java:366) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:240) Found 1 deadlock.