2014-09-03 13:41:19 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.45-b08 mixed mode): "IPC Client (238284366) connection to hadoopnnvip.cm6/10.246.10.153:9000 from hadoop" daemon prio=10 tid=0x00007f84c04e4000 nid=0x4698 in Object.wait() [0x000000006829f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:902) - locked <0x000000027f4e6470> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:947) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-9" prio=10 tid=0x00007f84bc02f800 nid=0x1c2 waiting on condition [0x0000000056e8c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-8" prio=10 tid=0x00007f84bc4cc000 nid=0x1c0 waiting on condition [0x0000000043f5e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-7" prio=10 tid=0x0000000004a0d800 nid=0x1bb waiting on condition [0x000000005ef0c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-6" prio=10 tid=0x00000000029af800 nid=0x1b8 waiting on condition [0x0000000057d9b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-5" prio=10 tid=0x00000000016af800 nid=0x1b6 waiting on condition [0x0000000045574000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-4" prio=10 tid=0x0000000004f40800 nid=0x1b5 waiting on condition [0x000000005bedc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-3" prio=10 tid=0x0000000000de8000 nid=0x1b4 waiting on condition [0x0000000045a79000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-2" prio=10 tid=0x0000000001079800 nid=0x1b3 waiting on condition [0x0000000045877000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "ResponseProcessor for block BP-66479029-10.246.10.12-1355889485474:blk_3170255677_1113477119167" daemon prio=10 tid=0x0000000000a5c000 nid=0x15d runnable [0x00000000595b3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000316939140> (a sun.nio.ch.Util$2) - locked <0x0000000316939128> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000315a07810> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:1986) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:796) Locked ownable synchronizers: - None "DataStreamer for file /hbase/WALs/hadoop461.cm6.tbsite.net,60020,1409003284950/hadoop461.cm6.tbsite.net%2C60020%2C1409003284950.1409722683780 block BP-66479029-10.246.10.12-1355889485474:blk_3170255677_1113477119167" daemon prio=10 tid=0x00007f84c4d44800 nid=0x15a in Object.wait() [0x000000006213e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:502) - locked <0x000000029caae600> (a java.util.LinkedList) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-1" prio=10 tid=0x0000000003618800 nid=0x7757 waiting on condition [0x0000000044968000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_CLOSE_REGION-hadoop461:60020-0" prio=10 tid=0x00000000029ca800 nid=0x76be waiting on condition [0x000000005a1bf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21b550> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Async Writer Thread 0 for [attachment=>spider_barcode_image_rowkey_queue]" daemon prio=10 tid=0x00000000012eb800 nid=0x5dd3 sleeping[0x000000005bddb000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 2 for [pagebase=>url_score_queue]" daemon prio=10 tid=0x00007f84c42da000 nid=0x5d2d sleeping[0x0000000067693000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 1 for [pagebase=>url_score_queue]" daemon prio=10 tid=0x00007f84c42cf800 nid=0x5d2c sleeping[0x000000005ee0b000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>url_score_queue]" daemon prio=10 tid=0x00007f84c434d000 nid=0x5d2b sleeping[0x0000000064562000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>url_fail_queue]" daemon prio=10 tid=0x00007f84c41eb000 nid=0x5d29 sleeping[0x000000005e502000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>srp_youhui_queue]" daemon prio=10 tid=0x00007f84c46ba000 nid=0x5d28 sleeping[0x0000000059ebc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>spider_redirect_input_queue]" daemon prio=10 tid=0x00007f84c4094000 nid=0x5d26 sleeping[0x00000000597b5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>spider_gain_rowkey_queue]" daemon prio=10 tid=0x00007f84c491b000 nid=0x5d1d sleeping[0x0000000059cba000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>promotion_rowkey_queue]" daemon prio=10 tid=0x00000000018de000 nid=0x5d1b sleeping[0x00000000599b7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 2 for [pagebase=>link_rowkey_queue]" daemon prio=10 tid=0x0000000005ab0000 nid=0x5d18 sleeping[0x0000000067895000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 1 for [pagebase=>link_rowkey_queue]" daemon prio=10 tid=0x0000000003ddb800 nid=0x5d17 sleeping[0x0000000070f2b000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>link_rowkey_queue]" daemon prio=10 tid=0x00000000006b4800 nid=0x5d16 sleeping[0x0000000064a67000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>flow_fast_circle_offline_inc]" daemon prio=10 tid=0x000000000071a000 nid=0x5d12 sleeping[0x000000006ec08000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>flow_fast_circle_hqueue]" daemon prio=10 tid=0x000000000a53d000 nid=0x5d11 sleeping[0x00000000687a4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>business_rowkey_queue]" daemon prio=10 tid=0x00000000015f7000 nid=0x5d10 sleeping[0x0000000064966000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 2 for [pagebase=>b2c_crawler_rowkey_queue]" daemon prio=10 tid=0x0000000000c9e000 nid=0x5d0f sleeping[0x00000000589a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 1 for [pagebase=>b2c_crawler_rowkey_queue]" daemon prio=10 tid=0x0000000001f23000 nid=0x5d0e sleeping[0x00000000587a5000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>b2c_crawler_rowkey_queue]" daemon prio=10 tid=0x0000000001025000 nid=0x5d0d sleeping[0x00000000588a6000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>attachment_sku_queue]" daemon prio=10 tid=0x0000000006898000 nid=0x5d0b sleeping[0x00000000586a4000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 2 for [pagebase=>attachment_rowkey_queue]" daemon prio=10 tid=0x0000000006a76000 nid=0x5d09 sleeping[0x0000000057c9a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 1 for [pagebase=>attachment_rowkey_queue]" daemon prio=10 tid=0x0000000001243000 nid=0x5d08 sleeping[0x0000000044a69000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>attachment_rowkey_queue]" daemon prio=10 tid=0x0000000000cad800 nid=0x5d07 sleeping[0x000000005c2e0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [pagebase=>amazon_crawler_to_feed_queue]" daemon prio=10 tid=0x0000000005a81800 nid=0x5d02 sleeping[0x0000000057b99000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.etao.hbase.coprocessor.increment.IncrementCoprocessor$AsyncWriterThread.run(IncrementCoprocessor.java:437) Locked ownable synchronizers: - None "Async Writer Thread 0 for [b2c_feed_extend=>attachment_rowkey_queue]-EventThread" daemon prio=10 tid=0x00007f84c4889800 nid=0x5a05 waiting on condition [0x000000005f613000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002c3945a28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "Async Writer Thread 0 for [b2c_feed_extend=>attachment_rowkey_queue]-SendThread(hadoop197.cm6.tbsite.net:2181)" daemon prio=10 tid=0x00007f84c41ec000 nid=0x5a04 runnable [0x0000000060320000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002c3a16dd8> (a sun.nio.ch.Util$2) - locked <0x00000002c3a16dc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002dde8df68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "Async Writer Thread 0 for [tba_buy_thread_reply=>wenda_tba_thread_reply_rowkey_queue]-EventThread" daemon prio=10 tid=0x0000000001395000 nid=0x280a waiting on condition [0x0000000067d9a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002e16fec20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "Async Writer Thread 0 for [tba_buy_thread_reply=>wenda_tba_thread_reply_rowkey_queue]-SendThread(hadoop194.cm6.tbsite.net:2181)" daemon prio=10 tid=0x00000000014de800 nid=0x2809 runnable [0x000000006dcf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002e16ff1b0> (a sun.nio.ch.Util$2) - locked <0x00000002e16ff198> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002d765f6a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "IPC Parameter Sending Thread #6" daemon prio=10 tid=0x00007f84c416a800 nid=0x3776 waiting on condition [0x000000006dbf8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bae1e2d0> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "pool-116-thread-1" prio=10 tid=0x00007f84d8133000 nid=0x7cd waiting on condition [0x000000006e804000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002c0cbc220> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-1-EventThread" daemon prio=10 tid=0x00007f84da966000 nid=0x7be waiting on condition [0x000000006d5f2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002c0ce71c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-1-SendThread(appzk3.cm6.tbsite.net:2181)" daemon prio=10 tid=0x00007f84d89d6800 nid=0x7bd runnable [0x0000000074f6b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002c0ce7068> (a sun.nio.ch.Util$2) - locked <0x00000002c0ce7050> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002c69a2b28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x00007f84d3011800 nid=0x21a0 waiting on condition [0x000000005ed0a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb028400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-1-EventThread" daemon prio=10 tid=0x0000000001a8b000 nid=0x2401 waiting on condition [0x0000000061633000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bbc81288> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-1-SendThread(appzk2.cm6.tbsite.net:2181)" daemon prio=10 tid=0x0000000000a78800 nid=0x2400 runnable [0x000000005cbe9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002bbc80c20> (a sun.nio.ch.Util$2) - locked <0x00000002bbc80c08> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002bbc7d3f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-9" prio=10 tid=0x00000000018a7000 nid=0x2003 waiting on condition [0x000000005d5f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-8" prio=10 tid=0x000000000161a800 nid=0x1e7f waiting on condition [0x000000006011e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-7" prio=10 tid=0x0000000001b7f800 nid=0x1e43 waiting on condition [0x0000000061b38000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-6" prio=10 tid=0x00000000008d1000 nid=0x1ce8 waiting on condition [0x0000000061d3a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_LOG_REPLAY_OPS-hadoop461:60020-0" prio=10 tid=0x00007f84d0323800 nid=0x190c waiting on condition [0x000000005c1df000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb214a18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-5-EventThread" daemon prio=10 tid=0x0000000000f33000 nid=0x1603 waiting on condition [0x000000005cae8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb8f6728> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-5-SendThread(appzk1.cm6.tbsite.net:2181)" daemon prio=10 tid=0x0000000000f32800 nid=0x1602 runnable [0x000000005bad8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002bb83f0c8> (a sun.nio.ch.Util$2) - locked <0x00000002bb83f0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002bb862258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-5" prio=10 tid=0x0000000000f31800 nid=0x15dd waiting on condition [0x000000005bbd9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-9" prio=10 tid=0x00007f84d0310800 nid=0xb57 waiting on condition [0x000000005c4e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-8" prio=10 tid=0x000000000134a800 nid=0xb56 waiting on condition [0x000000005c6e4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-7" prio=10 tid=0x0000000001346800 nid=0xb55 waiting on condition [0x000000005c5e3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-6" prio=10 tid=0x0000000001346000 nid=0xb54 waiting on condition [0x000000005c8e6000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-5" prio=10 tid=0x0000000000a90000 nid=0xb53 waiting on condition [0x000000005c9e7000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-4" prio=10 tid=0x0000000000a8f000 nid=0xb52 waiting on condition [0x000000005c7e5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-4" prio=10 tid=0x0000000000723000 nid=0xb40 waiting on condition [0x000000005bcda000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Increment Coprocessor Rules Reload Timer - attachment" prio=10 tid=0x0000000000f37800 nid=0xadc in Object.wait() [0x000000005b9d7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb83e1c0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Increment Coprocessor Monitor Timer - attachment" prio=10 tid=0x00000000012ec800 nid=0xadb in Object.wait() [0x000000005b7d5000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb911618> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-3" prio=10 tid=0x0000000000872000 nid=0xad2 waiting on condition [0x000000005b6d4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Copy Rules Reload Timer" prio=10 tid=0x00007f8501a9d000 nid=0xaab in Object.wait() [0x000000005b5d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb72acf8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Copy Rules Reload Timer" prio=10 tid=0x00007f84d866d000 nid=0xaaa in Object.wait() [0x000000005b4d2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb732550> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Compare Rules Reload Timer" prio=10 tid=0x00007f84d85e1800 nid=0xaa9 in Object.wait() [0x000000005b3d1000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb8121d0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Increment Coprocessor Rules Reload Timer - pagebase" prio=10 tid=0x00007f8501932000 nid=0xaa6 in Object.wait() [0x000000005b0ce000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb7f9008> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Increment Coprocessor Monitor Timer - pagebase" prio=10 tid=0x00007f85018ce800 nid=0xaa5 in Object.wait() [0x000000005afcd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bb767658> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-2" prio=10 tid=0x0000000000d2e800 nid=0xa3f waiting on condition [0x0000000044867000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x00000000012e9800 nid=0x9c2 waiting on condition [0x000000005809e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb028d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-3" prio=10 tid=0x0000000000f3b000 nid=0x684 waiting on condition [0x0000000057896000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-2" prio=10 tid=0x0000000000f3a800 nid=0x683 waiting on condition [0x0000000057f9d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-1" prio=10 tid=0x0000000000f39800 nid=0x682 waiting on condition [0x000000005819f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_PARALLEL_SEEK-hadoop461:60020-0" prio=10 tid=0x0000000000d07000 nid=0x681 waiting on condition [0x0000000057e9c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb215ae0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-1" prio=10 tid=0x0000000000cb0800 nid=0x66f waiting on condition [0x0000000057997000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-0-EventThread" daemon prio=10 tid=0x0000000000db3000 nid=0x61e waiting on condition [0x0000000057694000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bae97288> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-0-SendThread(appzk3.cm6.tbsite.net:2181)" daemon prio=10 tid=0x0000000000db2800 nid=0x61d runnable [0x0000000057593000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002bae881f0> (a sun.nio.ch.Util$2) - locked <0x00000002bae881d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002bb7090d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "RS_OPEN_REGION-hadoop461:60020-0" prio=10 tid=0x000000000075b800 nid=0x61c waiting on condition [0x0000000057492000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb21d980> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "org.apache.hadoop.hdfs.PeerCache@6c95609a" daemon prio=10 tid=0x0000000000921800 nid=0x61b waiting on condition [0x0000000057391000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:245) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:41) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:119) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "SplitLogWorker-hadoop461.cm6.tbsite.net,60020,1409003284950-EventThread" daemon prio=10 tid=0x00007f84d002f800 nid=0x7625 waiting on condition [0x0000000057290000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb221028> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "SplitLogWorker-hadoop461.cm6.tbsite.net,60020,1409003284950-SendThread(hadoop197.cm6.tbsite.net:2181)" daemon prio=10 tid=0x00007f84d002e800 nid=0x7624 runnable [0x000000005718f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002bb4ee208> (a sun.nio.ch.Util$2) - locked <0x00000002bb4ee1f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002bb3f6ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "hadoop461:60020Replication Statistics #0" daemon prio=10 tid=0x00007f8501369800 nid=0x7511 waiting on condition [0x0000000045f7e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bae0ff88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "regionserver60020-EventThread" daemon prio=10 tid=0x00007f8500c1e800 nid=0x7510 waiting on condition [0x0000000045e7d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bae0f538> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "regionserver60020-SendThread(hadoop191.cm6.tbsite.net:2181)" daemon prio=10 tid=0x00007f8500c1d000 nid=0x750f runnable [0x0000000045d7c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002bae20c10> (a sun.nio.ch.Util$2) - locked <0x00000002bae20c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002bae1fea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "ReplicationExecutor-0" prio=10 tid=0x00007f8500cef000 nid=0x750e waiting on condition [0x0000000045c7b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bae2b8e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "regionserver60020.leaseChecker" daemon prio=10 tid=0x00007f8500bb7800 nid=0x7507 sleeping[0x0000000045b7a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:95) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "HBase-Metrics2-1" daemon prio=10 tid=0x00007f8500241800 nid=0x7500 runnable [0x0000000045473000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bae089e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "LeaseRenewer:hadoop@hadoopnnvip.cm6:9000" daemon prio=10 tid=0x00007f8501540000 nid=0x74ea sleeping[0x0000000044b6a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "JvmPauseMonitor" daemon prio=10 tid=0x00000000008f6000 nid=0x74e2 sleeping[0x0000000044665000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:159) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Thread-12" daemon prio=10 tid=0x0000000000a6e800 nid=0x74d8 runnable [0x0000000044564000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$800(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$1.run(DomainSocketWatcher.java:457) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "regionserver60020-EventThread" daemon prio=10 tid=0x00007f84d86d8000 nid=0x74d2 waiting on condition [0x0000000044261000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000002bb4c39e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) Locked ownable synchronizers: - None "regionserver60020-SendThread(hadoop198.cm6.tbsite.net:2181)" daemon prio=10 tid=0x00007f84d8316000 nid=0x74d1 runnable [0x0000000044160000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000002bae20c70> (a sun.nio.ch.Util$2) - locked <0x00000002bae20c88> (a java.util.Collections$UnmodifiableSet) - locked <0x00000002bae1ffc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) Locked ownable synchronizers: - None "regionserver60020" prio=10 tid=0x00007f85011ca800 nid=0x74d0 in Object.wait() [0x000000004405f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.hadoop.hbase.util.DrainBarrier.stopAndDrainOps(DrainBarrier.java:115) - locked <0x00000002bb325248> (a org.apache.hadoop.hbase.util.DrainBarrier) at org.apache.hadoop.hbase.util.DrainBarrier.stopAndDrainOps(DrainBarrier.java:85) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.close(FSHLog.java:923) at org.apache.hadoop.hbase.regionserver.HRegionServer.closeWAL(HRegionServer.java:1208) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1001) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Timer for 'HBase' metrics system" daemon prio=10 tid=0x00007f8500fcf800 nid=0x74c9 in Object.wait() [0x0000000043a59000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000002bae1e3f0> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "ganglia2" daemon prio=10 tid=0x00007f8500fbf800 nid=0x74c8 in Object.wait() [0x0000000043958000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.hadoop.metrics2.impl.SinkQueue.waitForData(SinkQueue.java:114) - locked <0x00000002bae0b768> (a org.apache.hadoop.metrics2.impl.SinkQueue) at org.apache.hadoop.metrics2.impl.SinkQueue.consumeAll(SinkQueue.java:83) at org.apache.hadoop.metrics2.impl.MetricsSinkAdapter.publishMetricsFromQueue(MetricsSinkAdapter.java:127) at org.apache.hadoop.metrics2.impl.MetricsSinkAdapter$1.run(MetricsSinkAdapter.java:86) Locked ownable synchronizers: - None "ganglia1" daemon prio=10 tid=0x00007f8500fbe000 nid=0x74c7 in Object.wait() [0x0000000043857000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.hadoop.metrics2.impl.SinkQueue.waitForData(SinkQueue.java:114) - locked <0x00000002bae08698> (a org.apache.hadoop.metrics2.impl.SinkQueue) at org.apache.hadoop.metrics2.impl.SinkQueue.consumeAll(SinkQueue.java:83) at org.apache.hadoop.metrics2.impl.MetricsSinkAdapter.publishMetricsFromQueue(MetricsSinkAdapter.java:127) at org.apache.hadoop.metrics2.impl.MetricsSinkAdapter$1.run(MetricsSinkAdapter.java:86) Locked ownable synchronizers: - None "ganglia0" daemon prio=10 tid=0x00007f8500fb9800 nid=0x74c6 in Object.wait() [0x0000000043756000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.hadoop.metrics2.impl.SinkQueue.waitForData(SinkQueue.java:114) - locked <0x00000002bae0ae08> (a org.apache.hadoop.metrics2.impl.SinkQueue) at org.apache.hadoop.metrics2.impl.SinkQueue.consumeAll(SinkQueue.java:83) at org.apache.hadoop.metrics2.impl.MetricsSinkAdapter.publishMetricsFromQueue(MetricsSinkAdapter.java:127) at org.apache.hadoop.metrics2.impl.MetricsSinkAdapter$1.run(MetricsSinkAdapter.java:86) Locked ownable synchronizers: - None "Attach Listener" daemon prio=10 tid=0x000000000072f000 nid=0x74a5 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "RMI TCP Accept-0" daemon prio=10 tid=0x00007f8500a5e000 nid=0x74a3 runnable [0x0000000042a49000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) at java.net.ServerSocket.implAccept(ServerSocket.java:530) at java.net.ServerSocket.accept(ServerSocket.java:498) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:388) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:360) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RMI TCP Accept-10102" daemon prio=10 tid=0x00007f8500a54000 nid=0x74a2 runnable [0x0000000042948000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) at java.net.ServerSocket.implAccept(ServerSocket.java:530) at java.net.ServerSocket.accept(ServerSocket.java:498) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:388) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:360) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "RMI TCP Accept-0" daemon prio=10 tid=0x00007f8500a47000 nid=0x74a1 runnable [0x0000000042847000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) at java.net.ServerSocket.implAccept(ServerSocket.java:530) at java.net.ServerSocket.accept(ServerSocket.java:498) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:388) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:360) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - None "Service Thread" daemon prio=10 tid=0x00007f850023f800 nid=0x749d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" daemon prio=10 tid=0x00007f850023d000 nid=0x749c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" daemon prio=10 tid=0x00007f850023a800 nid=0x749b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" daemon prio=10 tid=0x00007f8500238800 nid=0x749a runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x00007f8500236800 nid=0x7499 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" daemon prio=10 tid=0x00007f8500217800 nid=0x7498 in Object.wait() [0x0000000042342000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000002bae32810> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189) Locked ownable synchronizers: - None "Reference Handler" daemon prio=10 tid=0x00007f8500213800 nid=0x7497 in Object.wait() [0x0000000041319000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x00000002bae007a0> (a java.lang.ref.Reference$Lock) Locked ownable synchronizers: - None "main" prio=10 tid=0x0000000000626800 nid=0x7476 in Object.wait() [0x0000000040aa1000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000002bae011c0> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1280) - locked <0x00000002bae011c0> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1354) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:64) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:85) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2437) Locked ownable synchronizers: - None "VM Thread" prio=10 tid=0x00007f8500211800 nid=0x7496 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x0000000000638000 nid=0x747e runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000000639800 nid=0x747f runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x000000000063b800 nid=0x7480 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x000000000063d800 nid=0x7481 runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x00007f8500001800 nid=0x7482 runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x00007f8500003800 nid=0x7483 runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x00007f8500005000 nid=0x7484 runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x00007f8500007000 nid=0x7485 runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x00007f8500009000 nid=0x7486 runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x00007f850000a800 nid=0x7487 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x00007f850000c800 nid=0x7488 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x00007f850000e800 nid=0x7489 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x00007f8500010800 nid=0x748a runnable "Gang worker#13 (Parallel GC Threads)" prio=10 tid=0x00007f8500012000 nid=0x748b runnable "Gang worker#14 (Parallel GC Threads)" prio=10 tid=0x00007f8500014000 nid=0x748c runnable "Gang worker#15 (Parallel GC Threads)" prio=10 tid=0x00007f8500016000 nid=0x748d runnable "Gang worker#16 (Parallel GC Threads)" prio=10 tid=0x00007f8500017800 nid=0x748e runnable "Gang worker#17 (Parallel GC Threads)" prio=10 tid=0x00007f8500019800 nid=0x748f runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00007f85001cd000 nid=0x7495 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x00007f85001c3000 nid=0x7490 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x00007f85001c4800 nid=0x7491 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x00007f85001c6800 nid=0x7492 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x00007f85001c8800 nid=0x7493 runnable "Gang worker#4 (Parallel CMS Threads)" prio=10 tid=0x00007f85001ca000 nid=0x7494 runnable "VM Periodic Task Thread" prio=10 tid=0x00007f8500a60000 nid=0x74a4 waiting on condition JNI global references: 582