2011-05-15 07:15:09 Full thread dump Java HotSpot(TM) 64-Bit Server VM (19.0-b09 mixed mode): "IPC Client (47) connection to hmaster1/192.168.40.100:9000 from flurry" daemon prio=10 tid=0x00002aaabc9d0000 nid=0x76e1 in Object.wait() [0x0000000043a1a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:403) - locked <0x00000006840146b8> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:445) "Attach Listener" daemon prio=10 tid=0x00000000556c8000 nid=0x76b6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ResponseProcessor for block blk_5971686596064896651_109624563" daemon prio=10 tid=0x000000005556c000 nid=0x7282 runnable [0x000000004122c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000731c9f8f8> (a sun.nio.ch.Util$2) - locked <0x0000000731c9f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000764e5a538> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:178) at java.io.DataInputStream.readLong(DataInputStream.java:399) at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:122) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2601) "DataStreamer for file /hbase/.logs/hslave127,60020,1305340082341/hslave127%3A60020.1305468572941 block blk_5971686596064896651_109624563" daemon prio=10 tid=0x00002aaac8052000 nid=0x727d in Object.wait() [0x0000000047757000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2438) - locked <0x00000006aa58a278> (a java.util.LinkedList) "IPC Client (47) connection to hmaster1/192.168.40.100:60000 from an unknown user" daemon prio=10 tid=0x00002aaac4a1f000 nid=0x711d in Object.wait() [0x0000000043818000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:431) - locked <0x000000073dd85800> (a org.apache.hadoop.hbase.ipc.HBaseClient$Connection) at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:476) "RS_OPEN_REGION-hslave127,60020,1305340082341-2" daemon prio=10 tid=0x0000000055491800 nid=0x3e3c waiting on condition [0x0000000043919000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000695720a28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_OPEN_REGION-hslave127,60020,1305340082341-1" daemon prio=10 tid=0x0000000055567000 nid=0x3e3b waiting on condition [0x0000000047454000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000695720a28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "RS_OPEN_REGION-hslave127,60020,1305340082341-0" daemon prio=10 tid=0x000000005516f000 nid=0x3e3a waiting on condition [0x0000000047555000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000695720a28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "LRU Statistics #0" prio=10 tid=0x00002aaac4323000 nid=0x3e1d waiting on condition [0x0000000047252000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000695720a10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:662) "LruBlockCache.EvictionThread" daemon prio=10 tid=0x00002aaac4321800 nid=0x3e1c in Object.wait() [0x0000000047151000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:519) - locked <0x0000000695765218> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) "PRI IPC Server handler 9 on 60020" daemon prio=10 tid=0x00002aaac431f800 nid=0x3e1b waiting on condition [0x0000000047050000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 8 on 60020" daemon prio=10 tid=0x00002aaac431d800 nid=0x3e1a waiting on condition [0x0000000046f4f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 7 on 60020" daemon prio=10 tid=0x00002aaac42aa000 nid=0x3e19 waiting on condition [0x0000000046e4e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 6 on 60020" daemon prio=10 tid=0x00002aaac42a8000 nid=0x3e18 waiting on condition [0x0000000046d4d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 5 on 60020" daemon prio=10 tid=0x00002aaac42a6000 nid=0x3e17 waiting on condition [0x0000000046c4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 4 on 60020" daemon prio=10 tid=0x00002aaac42a4000 nid=0x3e16 waiting on condition [0x0000000046b4b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 3 on 60020" daemon prio=10 tid=0x00002aaac42a2000 nid=0x3e15 waiting on condition [0x0000000046a4a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 2 on 60020" daemon prio=10 tid=0x00002aaac42a0000 nid=0x3e14 waiting on condition [0x0000000046949000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 1 on 60020" daemon prio=10 tid=0x00002aaac429e000 nid=0x3e13 waiting on condition [0x0000000046848000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "PRI IPC Server handler 0 on 60020" daemon prio=10 tid=0x00002aaac429c000 nid=0x3e12 waiting on condition [0x0000000046747000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd5fe8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025) "IPC Server handler 29 on 60020" daemon prio=10 tid=0x00002aaac402f800 nid=0x3e11 waiting for monitor entry [0x0000000046646000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 28 on 60020" daemon prio=10 tid=0x00002aaac402d800 nid=0x3e10 waiting for monitor entry [0x0000000046545000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 27 on 60020" daemon prio=10 tid=0x00002aaac402b800 nid=0x3e0f waiting for monitor entry [0x0000000046444000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 26 on 60020" daemon prio=10 tid=0x00002aaac4029800 nid=0x3e0e waiting for monitor entry [0x0000000046343000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 25 on 60020" daemon prio=10 tid=0x00002aaac419c000 nid=0x3e0d runnable [0x0000000046242000] java.lang.Thread.State: RUNNABLE at java.util.TreeMap.containsKey(TreeMap.java:1705) at java.util.TreeSet.contains(TreeSet.java:217) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2096) - locked <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 24 on 60020" daemon prio=10 tid=0x00002aaac419a000 nid=0x3e0c waiting for monitor entry [0x0000000046141000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 23 on 60020" daemon prio=10 tid=0x00002aaac4198000 nid=0x3e0b waiting for monitor entry [0x0000000046040000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 22 on 60020" daemon prio=10 tid=0x00002aaac4196000 nid=0x3e0a waiting for monitor entry [0x0000000045f3f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 21 on 60020" daemon prio=10 tid=0x00002aaac4174800 nid=0x3e09 waiting for monitor entry [0x0000000045e3e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 20 on 60020" daemon prio=10 tid=0x00002aaac4172800 nid=0x3e08 waiting for monitor entry [0x0000000045d3d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 19 on 60020" daemon prio=10 tid=0x00002aaac4170800 nid=0x3e07 waiting for monitor entry [0x0000000045c3c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 18 on 60020" daemon prio=10 tid=0x00002aaac416e800 nid=0x3e06 waiting for monitor entry [0x0000000045b3b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 17 on 60020" daemon prio=10 tid=0x00002aaac4281000 nid=0x3e05 waiting for monitor entry [0x0000000045a3a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 16 on 60020" daemon prio=10 tid=0x00002aaac427f000 nid=0x3e04 waiting for monitor entry [0x0000000045939000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 15 on 60020" daemon prio=10 tid=0x00002aaac427d000 nid=0x3e03 waiting for monitor entry [0x0000000045838000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 14 on 60020" daemon prio=10 tid=0x00002aaac4154800 nid=0x3e02 waiting for monitor entry [0x0000000045737000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 13 on 60020" daemon prio=10 tid=0x00002aaac4152800 nid=0x3e01 waiting for monitor entry [0x0000000045636000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 12 on 60020" daemon prio=10 tid=0x00002aaac4150800 nid=0x3e00 waiting for monitor entry [0x0000000045535000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 11 on 60020" daemon prio=10 tid=0x00002aaac42e6800 nid=0x3dff waiting for monitor entry [0x0000000045434000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 10 on 60020" daemon prio=10 tid=0x00002aaac42e4800 nid=0x3dfe waiting for monitor entry [0x0000000045333000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 9 on 60020" daemon prio=10 tid=0x00002aaac41db000 nid=0x3dfd waiting for monitor entry [0x0000000045232000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 8 on 60020" daemon prio=10 tid=0x00002aaac41d9000 nid=0x3dfc waiting for monitor entry [0x0000000045131000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 7 on 60020" daemon prio=10 tid=0x00002aaac41d7800 nid=0x3dfb waiting for monitor entry [0x0000000045030000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 6 on 60020" daemon prio=10 tid=0x00002aaac4302800 nid=0x3dfa waiting for monitor entry [0x0000000044f2f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 5 on 60020" daemon prio=10 tid=0x00002aaac4300800 nid=0x3df9 waiting for monitor entry [0x0000000044e2e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 4 on 60020" daemon prio=10 tid=0x00002aaac4179000 nid=0x3df8 waiting for monitor entry [0x0000000044d2d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 3 on 60020" daemon prio=10 tid=0x00002aaac4177800 nid=0x3df7 waiting for monitor entry [0x0000000044c2c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 2 on 60020" daemon prio=10 tid=0x00002aaac4123800 nid=0x3df6 waiting for monitor entry [0x0000000044b2b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.releaseRowLock(HRegion.java:2148) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1498) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 1 on 60020" daemon prio=10 tid=0x00002aaac4207800 nid=0x3df5 waiting for monitor entry [0x0000000044a2a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server handler 0 on 60020" daemon prio=10 tid=0x00002aaac4206800 nid=0x3df4 waiting for monitor entry [0x0000000044929000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2095) - waiting to lock <0x00000006969920e8> (a java.util.TreeSet) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2533) at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039) "IPC Server listener on 60020" daemon prio=10 tid=0x00002aaac42b9800 nid=0x3df3 waiting for monitor entry [0x0000000044828000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.registerChannel(HBaseServer.java:347) - waiting to lock <0x0000000694c62fe8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doAccept(HBaseServer.java:496) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:422) "IPC Server Responder" daemon prio=10 tid=0x00002aaac423b800 nid=0x3df2 runnable [0x0000000044727000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694ccf6b0> (a sun.nio.ch.Util$2) - locked <0x0000000694ccf698> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c93db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:588) "Timer-0" daemon prio=10 tid=0x0000000055470000 nid=0x3deb in Object.wait() [0x0000000044626000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x0000000694cb4f78> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "160194151@qtp-713849829-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030" prio=10 tid=0x00000000553fe000 nid=0x3dea runnable [0x0000000044525000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00000006957209b0> (a sun.nio.ch.Util$2) - locked <0x0000000695720968> (a java.util.Collections$UnmodifiableSet) - locked <0x00000006963f8690> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "46712790@qtp-713849829-0" prio=10 tid=0x0000000055400000 nid=0x3de9 in Object.wait() [0x0000000044424000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x00000006963f7628> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "regionserver60020.leaseChecker" daemon prio=10 tid=0x00002aaac42b2800 nid=0x3de8 waiting on condition [0x0000000044323000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cb4fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:209) at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:82) "regionserver60020.majorCompactionChecker" daemon prio=10 tid=0x00002aaac41f1800 nid=0x3de7 in Object.wait() [0x0000000044222000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) - locked <0x00000006963f66c8> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:74) "regionserver60020.compactor" daemon prio=10 tid=0x00002aaac41ef800 nid=0x3de6 waiting on condition [0x0000000044121000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cb4db0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.PriorityBlockingQueue.poll(PriorityBlockingQueue.java:245) at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.poll(PriorityCompactionQueue.java:248) at org.apache.hadoop.hbase.regionserver.CompactSplitThread.run(CompactSplitThread.java:75) "regionserver60020.cacheFlusher" daemon prio=10 tid=0x00002aaac41ed800 nid=0x3de5 waiting on condition [0x0000000044020000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006963f66e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:201) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:39) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:216) "regionserver60020.logRoller" daemon prio=10 tid=0x00002aaac41e9800 nid=0x3de4 in Object.wait() [0x0000000043f1f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:78) - locked <0x0000000694cb4ff0> (a java.util.concurrent.atomic.AtomicBoolean) "Timer thread for monitoring jvm" daemon prio=10 tid=0x00002aaac41e8000 nid=0x3de3 in Object.wait() [0x0000000043e1e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x0000000694cb5008> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "Timer thread for monitoring hbase" daemon prio=10 tid=0x00002aaac41e6800 nid=0x3de2 in Object.wait() [0x0000000043d1d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x00000006963f6698> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "regionserver60020.logSyncer" daemon prio=10 tid=0x00002aaac419e000 nid=0x3de1 waiting on condition [0x0000000043c1c000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:963) "LeaseChecker" daemon prio=10 tid=0x0000000055249000 nid=0x3de0 waiting on condition [0x0000000043b1b000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1112) at java.lang.Thread.run(Thread.java:662) "DestroyJavaVM" prio=10 tid=0x0000000054e23000 nid=0x3da3 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "regionserver60020-EventThread" daemon prio=10 tid=0x000000005521a000 nid=0x3dd1 waiting on condition [0x0000000043717000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694c608b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) "regionserver60020-SendThread(hslave1:2181)" daemon prio=10 tid=0x0000000055402000 nid=0x3dd0 runnable [0x0000000043616000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694c60900> (a sun.nio.ch.Util$2) - locked <0x0000000694c60918> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c62028> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107) "regionserver60020" prio=10 tid=0x00002aaabc3e2800 nid=0x3dcf waiting on condition [0x0000000043515000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006956fc1b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:424) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:622) at java.lang.Thread.run(Thread.java:662) "IPC Reader 9 on port 60020" prio=10 tid=0x00002aaabc356800 nid=0x3dce waiting on condition [0x0000000043414000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd6930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x0000000694c60960> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 8 on port 60020" prio=10 tid=0x00002aaabc354800 nid=0x3dcd runnable [0x0000000043313000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694c60738> (a sun.nio.ch.Util$2) - locked <0x0000000694c60750> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c65118> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:305) - locked <0x0000000694c60768> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 7 on port 60020" prio=10 tid=0x00002aaabc352800 nid=0x3dcc waiting on condition [0x0000000043212000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd6930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x0000000694c609c0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 6 on port 60020" prio=10 tid=0x00002aaabc330800 nid=0x3dcb waiting on condition [0x0000000043111000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd6930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x0000000694c666c8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 5 on port 60020" prio=10 tid=0x00002aaabc32e800 nid=0x3dca runnable [0x0000000043010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694c609f0> (a sun.nio.ch.Util$2) - locked <0x0000000694c60a08> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c62100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:305) - locked <0x0000000694c65f80> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 4 on port 60020" prio=10 tid=0x00002aaabc3b3800 nid=0x3dc9 waiting on condition [0x0000000042f0f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd6930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x0000000694c62fe8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 3 on port 60020" prio=10 tid=0x00002aaabc348800 nid=0x3dc8 waiting on condition [0x0000000042e0e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd6930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x0000000694c607c8> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 2 on port 60020" prio=10 tid=0x00002aaabc348000 nid=0x3dc7 waiting on condition [0x0000000042d0d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694cd6930> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985) at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316) - locked <0x0000000694c60828> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 1 on port 60020" prio=10 tid=0x00002aaabc2ba000 nid=0x3dc6 runnable [0x0000000042c0c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694c60858> (a sun.nio.ch.Util$2) - locked <0x0000000694c60870> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c651f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:305) - locked <0x0000000694c60888> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "IPC Reader 0 on port 60020" prio=10 tid=0x00002aaabc384800 nid=0x3dc5 runnable [0x000000004112b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694c65fb0> (a sun.nio.ch.Util$2) - locked <0x0000000694c65fc8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c62148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:305) - locked <0x0000000694c65fe0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) at java.lang.Thread.run(Thread.java:662) "main-EventThread" daemon prio=10 tid=0x00002aaabc399800 nid=0x3dc1 waiting on condition [0x0000000040fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000694c63018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObjectat java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) "main-SendThread(hslave1:2181)" daemon prio=10 tid=0x00002aaabc2a2800 nid=0x3dc0 runnable [0x0000000040818000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x0000000694c66010> (a sun.nio.ch.Util$2) - locked <0x0000000694c66028> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000694c62190> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107) "Low Memory Detector" daemon prio=10 tid=0x00002aaabc006000 nid=0x3dbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00002aaabc003800 nid=0x3dbc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x00002aaabc001000 nid=0x3dbb waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000551a8800 nid=0x3dba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000551a6800 nid=0x3db9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000055187800 nid=0x3db8 in Object.wait() [0x0000000042505000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x0000000694c66040> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x0000000055180000 nid=0x3db7 in Object.wait() [0x0000000042404000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x0000000694c62f88> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x000000005517b800 nid=0x3db6 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x0000000054e31800 nid=0x3da4 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000054e33800 nid=0x3da5 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000054e35000 nid=0x3da6 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000054e37000 nid=0x3da7 runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000054e39000 nid=0x3da8 runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000054e3a800 nid=0x3da9 runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000054e3c800 nid=0x3daa runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x0000000054e3e800 nid=0x3dab runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x0000000054e40000 nid=0x3dac runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x0000000054e42000 nid=0x3dad runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000054e44000 nid=0x3dae runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000054e45800 nid=0x3daf runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000054e47800 nid=0x3db0 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x0000000054fb8000 nid=0x3db5 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000054fb0800 nid=0x3db1 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000054fb2000 nid=0x3db2 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000054fb4000 nid=0x3db3 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x0000000054fb6000 nid=0x3db4 runnable "VM Periodic Task Thread" prio=10 tid=0x00002aaabc011800 nid=0x3dbe waiting on condition JNI global references: 1065 [flurry@hslave127 hbase]$