2013-02-16 08:17:03 Full thread dump Java HotSpot(TM) 64-Bit Server VM (23.1-b03 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000002d85800 nid=0x4111 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RS_CLOSE_META-node3,60020,1360977975359-0" prio=10 tid=0x0000000002924800 nid=0x3a4d waiting on condition [0x00007f74f32d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000624473920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node3,60020,1360977975359-2" prio=10 tid=0x00000000032ff000 nid=0x3a2d waiting on condition [0x00007f74f0ead000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000625b93ee8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.hadoop.hbase.regionserver.wal.HLog.startCacheFlush(HLog.java:1551) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1490) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1435) at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:968) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:916) - locked <0x0000000627dee918> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:119) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node3,60020,1360977975359-1" prio=10 tid=0x0000000002aab800 nid=0x3a2c waiting on condition [0x00007f74f1dbc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000625b93ee8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.hadoop.hbase.regionserver.wal.HLog.startCacheFlush(HLog.java:1551) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1490) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1435) at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:968) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:916) - locked <0x0000000627aecbe0> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:119) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "ResponseProcessor for block blk_-620121737711819419_330113" daemon prio=10 tid=0x0000000002ee6000 nid=0x32f9 runnable [0x00007f74f02a1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x000000062a40e508> (a sun.nio.ch.Util$2) - locked <0x000000062a40e4f0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000062a3a2650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:195) at java.io.DataInputStream.readLong(DataInputStream.java:416) at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:124) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2967) "DataStreamer for file /hbase/.logs/node3,60020,1360977975359/node3%2C60020%2C1360977975359.1361017579772 block blk_-620121737711819419_330113" daemon prio=10 tid=0x0000000003040000 nid=0x324b in Object.wait() [0x00007f74f33d2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2806) - locked <0x0000000627dfd340> (a java.util.LinkedList) "RS_CLOSE_REGION-node3,60020,1360977975359-0" prio=10 tid=0x000000000279f800 nid=0x1dc2 waiting on condition [0x00007f74f26c5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006244dbf98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node3,60020,1360977975359-2" prio=10 tid=0x0000000002845800 nid=0x18fe waiting on condition [0x00007f74f21c0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006244dbcb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node3,60020,1360977975359-1" prio=10 tid=0x0000000002844800 nid=0x18fd waiting on condition [0x00007f74ef493000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006244dbcb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node3,60020,1360977975359-0" prio=10 tid=0x0000000002844000 nid=0x18fc waiting on condition [0x00007f74f19b8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006244dbcb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_META-node3,60020,1360977975359-0" prio=10 tid=0x000000000343d000 nid=0x18e5 waiting on condition [0x00007f74f2ccb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006244dbe30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "Timer thread for monitoring jvm" daemon prio=10 tid=0x0000000002d6d000 nid=0x1755 in Object.wait() [0x00007f74f5e32000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000624c06a80> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "Timer thread for monitoring hbase" daemon prio=10 tid=0x0000000002d68800 nid=0x1754 in Object.wait() [0x00007f74f5f33000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000624bfd820> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "regionserver60020.logSyncer" daemon prio=10 tid=0x0000000002d60800 nid=0x1753 in Object.wait() [0x00007f74f6034000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1230) - locked <0x0000000624bfd730> (a java.util.concurrent.atomic.AtomicBoolean) at java.lang.Thread.run(Thread.java:722) "LeaseChecker" daemon prio=10 tid=0x0000000002d4d800 nid=0x1752 waiting on condition [0x00007f74f6135000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1379) at java.lang.Thread.run(Thread.java:722) "DestroyJavaVM" prio=10 tid=0x0000000002413000 nid=0x1712 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "regionserver60020-EventThread" daemon prio=10 tid=0x0000000002cab800 nid=0x173d waiting on condition [0x00007f74f6539000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006243d4b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver60020-SendThread(cube:2181)" daemon prio=10 tid=0x0000000002caa800 nid=0x173c runnable [0x00007f74f663a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000624411e68> (a sun.nio.ch.Util$2) - locked <0x0000000624411e80> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000624439440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "regionserver60020" prio=10 tid=0x0000000002c7c000 nid=0x173a waiting on condition [0x00007f74f693c000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:131) at org.apache.hadoop.hbase.regionserver.HRegionServer.waitOnAllRegionsToClose(HRegionServer.java:958) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:820) at java.lang.Thread.run(Thread.java:722) "Timer thread for monitoring rpc" daemon prio=10 tid=0x00007f74f8603800 nid=0x1737 in Object.wait() [0x00007f74f6c62000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000006243e9240> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "Service Thread" daemon prio=10 tid=0x00007f74f8006000 nid=0x1725 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00007f74f8003800 nid=0x1724 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x00007f74f8001000 nid=0x1723 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000025ef800 nid=0x1722 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x00000000025ed800 nid=0x1721 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000025a0800 nid=0x1720 in Object.wait() [0x00007f74fc900000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x0000000624412528> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:177) "Reference Handler" daemon prio=10 tid=0x0000000002599000 nid=0x171f in Object.wait() [0x00007f74fca01000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x0000000624412120> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x0000000002591800 nid=0x171e runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000000241c000 nid=0x1713 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x000000000241e000 nid=0x1714 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x000000000241f800 nid=0x1715 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000002421800 nid=0x1716 runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000002423800 nid=0x1717 runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000002425000 nid=0x1718 runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000002427000 nid=0x1719 runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x0000000002429000 nid=0x171a runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000000251c800 nid=0x171d runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000002518800 nid=0x171b runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x000000000251a800 nid=0x171c runnable "VM Periodic Task Thread" prio=10 tid=0x00007f74f8019800 nid=0x1726 waiting on condition JNI global references: 216