2013-02-16 08:16:58 Full thread dump Java HotSpot(TM) Client VM (23.1-b03 mixed mode): "Attach Listener" daemon prio=10 tid=0x09dac000 nid=0x5782 runnable [0x00000000] java.lang.Thread.State: RUNNABLE "ResponseProcessor for block blk_-1954400284530576549_330113" daemon prio=10 tid=0x09c16400 nid=0x50b4 runnable [0x2a818000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x3a0a7f88> (a sun.nio.ch.Util$2) - locked <0x3a0a7f98> (a java.util.Collections$UnmodifiableSet) - locked <0x3a2f27f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:195) at java.io.DataInputStream.readLong(DataInputStream.java:416) at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:124) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2967) "DataStreamer for file /hbase/.logs/node1,60020,1360977976739/node1%2C60020%2C1360977976739.1361017580689 block blk_-1954400284530576549_330113" daemon prio=10 tid=0x0a030400 nid=0x5003 in Object.wait() [0x296f8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2806) - locked <0x3a3340f8> (a java.util.LinkedList) "RS_CLOSE_REGION-node1,60020,1360977976739-2" prio=10 tid=0x09ef2400 nid=0x4bca waiting on condition [0x2992f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x39e1be88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node1,60020,1360977976739-1" prio=10 tid=0x09ef1800 nid=0x4bc5 waiting on condition [0x298de000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x39e09d40> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.hadoop.hbase.regionserver.wal.HLog.startCacheFlush(HLog.java:1551) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1490) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1435) at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:952) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:916) - locked <0x3a3cf050> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:119) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node1,60020,1360977976739-0" prio=10 tid=0x09cc1000 nid=0x4bba waiting on condition [0x299d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x39e09d40> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.hadoop.hbase.regionserver.wal.HLog.startCacheFlush(HLog.java:1551) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1490) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1435) at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:952) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:916) - locked <0x3a41f2a0> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:119) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node1,60020,1360977976739-2" prio=10 tid=0xb484bc00 nid=0x4692 waiting on condition [0x29a22000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x39e1c218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node1,60020,1360977976739-1" prio=10 tid=0x09bb6000 nid=0x4691 waiting on condition [0x29a73000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x39e1c218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node1,60020,1360977976739-0" prio=10 tid=0x09bb1800 nid=0x4690 waiting on condition [0x29ac4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x39e1c218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "Timer thread for monitoring jvm" daemon prio=10 tid=0x09da1800 nid=0x45c7 in Object.wait() [0x2a6d4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x39e24b50> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "Timer thread for monitoring hbase" daemon prio=10 tid=0x09d90c00 nid=0x45c6 in Object.wait() [0x2a725000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x39e24b70> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "regionserver60020.logSyncer" daemon prio=10 tid=0x09d79c00 nid=0x45c5 in Object.wait() [0x2a776000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1230) - locked <0x38e6f400> (a java.util.concurrent.atomic.AtomicBoolean) at java.lang.Thread.run(Thread.java:722) "LeaseChecker" daemon prio=10 tid=0x09d69000 nid=0x45c2 waiting on condition [0x2a7c7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1379) at java.lang.Thread.run(Thread.java:722) "DestroyJavaVM" prio=10 tid=0x2aa53c00 nid=0x45a0 waiting on condition [0x00000000] java.lang.Thread.State: RUNNABLE "regionserver60020-EventThread" daemon prio=10 tid=0x09d33c00 nid=0x45bc waiting on condition [0x2a90b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x38939140> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver60020-SendThread(cube:2181)" daemon prio=10 tid=0x09d35c00 nid=0x45bb runnable [0x2a95c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x38903690> (a sun.nio.ch.Util$2) - locked <0x389036a0> (a java.util.Collections$UnmodifiableSet) - locked <0x38936118> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "regionserver60020" prio=10 tid=0x2aefd400 nid=0x45ba waiting on condition [0x2a9ad000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:131) at org.apache.hadoop.hbase.regionserver.HRegionServer.waitOnAllRegionsToClose(HRegionServer.java:958) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:820) at java.lang.Thread.run(Thread.java:722) "Timer thread for monitoring rpc" daemon prio=10 tid=0x2ae76800 nid=0x45b7 in Object.wait() [0x2abc7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x38837d18> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "Service Thread" daemon prio=10 tid=0xb4800c00 nid=0x45aa runnable [0x00000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread0" daemon prio=10 tid=0x09ba4000 nid=0x45a9 waiting on condition [0x00000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x09ba2800 nid=0x45a8 runnable [0x00000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x09ba0c00 nid=0x45a7 waiting on condition [0x00000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x09b66800 nid=0x45a6 in Object.wait() [0x2ba2d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x38771a20> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:177) "Reference Handler" daemon prio=10 tid=0x09b61800 nid=0x45a5 in Object.wait() [0x2ba7e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x38771a30> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x09b5c000 nid=0x45a4 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x09a58c00 nid=0x45a1 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x09a5a400 nid=0x45a2 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x09acdc00 nid=0x45a3 runnable "VM Periodic Task Thread" prio=10 tid=0xb480b400 nid=0x45ab waiting on condition JNI global references: 201