2013-02-16 08:16:48 Full thread dump Java HotSpot(TM) 64-Bit Server VM (23.1-b03 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000002b2c800 nid=0x5ba5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RS_CLOSE_ROOT-node7,60020,1360977976102-0" prio=10 tid=0x00007f79f05cd800 nid=0x5aae waiting on condition [0x00007f79e6ae9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007113a2748> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node7,60020,1360977976102-2" prio=10 tid=0x00007f79f0347800 nid=0x5aa9 waiting on condition [0x00007f79efbf9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000710d0ab88> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.hadoop.hbase.regionserver.wal.HLog.startCacheFlush(HLog.java:1551) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1490) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1435) at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:952) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:916) - locked <0x000000071364cbb0> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:119) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node7,60020,1360977976102-1" prio=10 tid=0x00007f79f0527800 nid=0x5aa8 waiting on condition [0x00007f79e66e5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007113a2610> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_CLOSE_REGION-node7,60020,1360977976102-0" prio=10 tid=0x00007f79f0579000 nid=0x5aa7 waiting on condition [0x00007f79e53d2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000710d0ab88> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.hadoop.hbase.regionserver.wal.HLog.startCacheFlush(HLog.java:1551) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1490) at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:1435) at org.apache.hadoop.hbase.regionserver.HRegion.doClose(HRegion.java:968) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:916) - locked <0x0000000711d5e8f8> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler.process(CloseRegionHandler.java:119) at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:169) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "ResponseProcessor for block blk_2592223112222348860_330113" daemon prio=10 tid=0x0000000002814000 nid=0x54d7 runnable [0x00007f79e61e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000007135f28c8> (a sun.nio.ch.Util$2) - locked <0x00000007135f28b0> (a java.util.Collections$UnmodifiableSet) - locked <0x000000070fb04c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.DataInputStream.readFully(DataInputStream.java:195) at java.io.DataInputStream.readLong(DataInputStream.java:416) at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:124) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2967) "DataStreamer for file /hbase/.logs/node7,60020,1360977976102/node7%2C60020%2C1360977976102.1361017579869 block blk_2592223112222348860_330113" daemon prio=10 tid=0x00000000027c9000 nid=0x5475 in Object.wait() [0x00007f79ec829000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2806) - locked <0x00000007116da730> (a java.util.LinkedList) "RS_OPEN_REGION-node7,60020,1360977976102-2" prio=10 tid=0x00000000024f3800 nid=0x748b waiting on condition [0x00007f79e5edd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007113a2268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node7,60020,1360977976102-1" prio=10 tid=0x00000000024f2800 nid=0x748a waiting on condition [0x00007f79ed536000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007113a2268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_REGION-node7,60020,1360977976102-0" prio=10 tid=0x00000000024f1000 nid=0x7489 waiting on condition [0x00007f79ec92a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007113a2268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "RS_OPEN_ROOT-node7,60020,1360977976102-0" prio=10 tid=0x0000000002778800 nid=0x7481 waiting on condition [0x00007f79ec526000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007113a23a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1043) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1103) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:722) "Timer thread for monitoring jvm" daemon prio=10 tid=0x00007f79f05c3000 nid=0x72df in Object.wait() [0x00007f79ef7f6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007112257b8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "Timer thread for monitoring hbase" daemon prio=10 tid=0x00007f79f059f800 nid=0x72de in Object.wait() [0x00007f79ef8f7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000071121d190> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "regionserver60020.logSyncer" daemon prio=10 tid=0x00007f79f0801800 nid=0x72dd in Object.wait() [0x00007f79ef9f8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1230) - locked <0x0000000711225878> (a java.util.concurrent.atomic.AtomicBoolean) at java.lang.Thread.run(Thread.java:722) "LeaseChecker" daemon prio=10 tid=0x00007f79f073e800 nid=0x72dc waiting on condition [0x00007f79efaf9000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1379) at java.lang.Thread.run(Thread.java:722) "DestroyJavaVM" prio=10 tid=0x00007f79e806c000 nid=0x72b9 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "regionserver60020-EventThread" daemon prio=10 tid=0x00007f79f064b000 nid=0x72d8 waiting on condition [0x00007f79efefd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000070fd030b8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver60020-SendThread(node3:2181)" daemon prio=10 tid=0x00007f79f04c0000 nid=0x72d7 runnable [0x00007f79efffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:228) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:81) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000710754d80> (a sun.nio.ch.Util$2) - locked <0x0000000710250960> (a java.util.Collections$UnmodifiableSet) - locked <0x000000070fd2aa18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "regionserver60020" prio=10 tid=0x0000000002b36000 nid=0x72d6 waiting on condition [0x00007f79f4326000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:131) at org.apache.hadoop.hbase.regionserver.HRegionServer.waitOnAllRegionsToClose(HRegionServer.java:958) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:820) at java.lang.Thread.run(Thread.java:722) "Timer thread for monitoring rpc" daemon prio=10 tid=0x0000000002702800 nid=0x72d3 in Object.wait() [0x00007f79f4653000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x000000070fd03028> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "Service Thread" daemon prio=10 tid=0x00007f79f0006000 nid=0x72c6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00007f79f0003000 nid=0x72c5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x00007f79f0001000 nid=0x72c4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000023c6000 nid=0x72c3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x00000000023c4000 nid=0x72c2 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000002372000 nid=0x72c1 in Object.wait() [0x00007f79f62a8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000070fb3b6d0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:177) "Reference Handler" daemon prio=10 tid=0x000000000236f800 nid=0x72c0 in Object.wait() [0x00007f79f63a9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000070fb3b6b8> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x0000000002368000 nid=0x72bf runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000000225f000 nid=0x72ba runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000002261000 nid=0x72bb runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000002262800 nid=0x72bc runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000002264800 nid=0x72bd runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00000000022f3800 nid=0x72be runnable "VM Periodic Task Thread" prio=10 tid=0x00007f79f0021800 nid=0x72c7 waiting on condition JNI global references: 179