2016-02-11 11:21:40 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:42 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:43 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:43 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:44 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:45 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:46 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:47 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:47 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:48 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:49 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:21:50 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:23:45 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:23:46 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281 2016-02-11 11:23:47 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.75-b04 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000008a313000 nid=0xb422 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-225" prio=10 tid=0x00000000a6309000 nid=0xde08 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-218" prio=10 tid=0x000000009ea3c800 nid=0xddfd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-145" prio=10 tid=0x000000000ba4e000 nid=0xddce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-139" prio=10 tid=0x00000000039e1800 nid=0xddbb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-129" prio=10 tid=0x000000000991e800 nid=0xdde5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-128" prio=10 tid=0x000000001f16b800 nid=0xddd7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-122" prio=10 tid=0x0000000040e27000 nid=0xddc6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-117" prio=10 tid=0x000000000991c000 nid=0xddc3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-111" prio=10 tid=0x000000006d04a800 nid=0xddd2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-108" prio=10 tid=0x000000000a4f1000 nid=0xddc1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-96" prio=10 tid=0x000000008a314800 nid=0xdde7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-95" prio=10 tid=0x000000000e430000 nid=0xddb9 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-91" prio=10 tid=0x00000000091bd000 nid=0xddca runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-90" prio=10 tid=0x00000000091bc800 nid=0xddbf runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-86" prio=10 tid=0x000000001f16a000 nid=0xdddd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-85" prio=10 tid=0x000000008a312000 nid=0xddc5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-80" prio=10 tid=0x000000000e433000 nid=0xddc0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-79" prio=10 tid=0x000000000a316000 nid=0xddbd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-73" prio=10 tid=0x0000000008f08000 nid=0xddbe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-65" prio=10 tid=0x0000000008f0a800 nid=0xddba runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-64" prio=10 tid=0x000000000a4f2800 nid=0xddd3 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-61" prio=10 tid=0x000000000d972800 nid=0xddd6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-60" prio=10 tid=0x000000006d049000 nid=0xddbc runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-56" prio=10 tid=0x000000006d049800 nid=0xdde8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-55" prio=10 tid=0x000000000a4f3000 nid=0xddc8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-52" prio=10 tid=0x0000000079fe2000 nid=0xddd0 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-49" prio=10 tid=0x000000000ba50800 nid=0xddc7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-48" prio=10 tid=0x000000000a372000 nid=0xddc4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-45" prio=10 tid=0x000000000bf46800 nid=0xddd4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-44" prio=10 tid=0x000000002349f000 nid=0xddcd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-39" prio=10 tid=0x000000000a7a6000 nid=0xddb8 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-38" prio=10 tid=0x0000000007b2f000 nid=0xddb7 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-37" prio=10 tid=0x0000000009348000 nid=0xddb6 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-36" prio=10 tid=0x000000000a22f800 nid=0xddb5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ShortCircuitCache_SlotReleaser" daemon prio=10 tid=0x0000000008ca1000 nid=0xeeae waiting on condition [0x00007fca69078000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000426091b38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ShortCircuitCache_Cleaner" daemon prio=10 tid=0x0000000007652000 nid=0xe87b waiting on condition [0x00007fc9de06d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004260924c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@5a496ee3" daemon prio=10 tid=0x0000000007653800 nid=0xe87a waiting on condition [0x00007fc9de16e000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.lang.Thread.run(Thread.java:745) "Thread-23" prio=10 tid=0x000000000a4f0000 nid=0xdd8d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-22" prio=10 tid=0x000000000dec9000 nid=0xdd8f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-21" prio=10 tid=0x000000000dec8800 nid=0xdd91 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-20" prio=10 tid=0x000000000dec7800 nid=0xdd90 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-19" prio=10 tid=0x000000000dec7000 nid=0xdd8b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-18" prio=10 tid=0x000000000dec6000 nid=0xdd95 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-17" prio=10 tid=0x0000000009bbf000 nid=0xdd94 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-16" prio=10 tid=0x0000000009bbe800 nid=0xdd8c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-15" prio=10 tid=0x0000000009bbd800 nid=0xdd92 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-14" prio=10 tid=0x0000000009bbd000 nid=0xdd93 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-13" prio=10 tid=0x0000000009bbc000 nid=0xdd8e runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-11" prio=10 tid=0x000000000a22f000 nid=0xddb4 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Thread-6" prio=10 tid=0x0000000007b2e000 nid=0xddb2 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "main-EventThread" daemon prio=10 tid=0x0000000007dd4000 nid=0xdd60 waiting on condition [0x00007fca7aaa1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004261eb378> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(XXXXX.XXXXX.XXXXX.XXXXX:2181)" daemon prio=10 tid=0x0000000007afd800 nid=0xdd5f runnable [0x00007fca7aba2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000426290c20> (a sun.nio.ch.Util$2) - locked <0x0000000426290c10> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000426290988> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-3" daemon prio=10 tid=0x0000000007b31000 nid=0xdd5d runnable [0x00007fca7aca3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=10 tid=0x0000000007b2f800 nid=0xdd58 in Object.wait() [0x00007fca7b3dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x00000004263220b0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3040) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00000000075c3800 nid=0xdd4c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00000000075c4800 nid=0xdd4b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x0000000003827800 nid=0xdd4a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x0000000003828800 nid=0xdd49 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x0000000003829000 nid=0xdd48 in Object.wait() [0x00007fca8284f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x000000042652bb48> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" daemon prio=10 tid=0x0000000003826000 nid=0xdd47 in Object.wait() [0x00007fca82950000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x000000042652b7d8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x0000000003827000 nid=0xdcfb runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" prio=10 tid=0x0000000003992800 nid=0xdd46 runnable "GC task thread#0 (ParallelGC)" prio=10 tid=0x000000000381b000 nid=0xdd34 runnable "GC task thread#1 (ParallelGC)" prio=10 tid=0x000000000381a000 nid=0xdd35 runnable "GC task thread#2 (ParallelGC)" prio=10 tid=0x000000000381c800 nid=0xdd36 runnable "GC task thread#3 (ParallelGC)" prio=10 tid=0x000000000381c000 nid=0xdd37 runnable "GC task thread#4 (ParallelGC)" prio=10 tid=0x000000000381b800 nid=0xdd38 runnable "GC task thread#5 (ParallelGC)" prio=10 tid=0x000000000381d800 nid=0xdd39 runnable "GC task thread#6 (ParallelGC)" prio=10 tid=0x0000000003866800 nid=0xdd3a runnable "GC task thread#7 (ParallelGC)" prio=10 tid=0x0000000003866000 nid=0xdd3b runnable "GC task thread#8 (ParallelGC)" prio=10 tid=0x0000000003865800 nid=0xdd3c runnable "GC task thread#9 (ParallelGC)" prio=10 tid=0x0000000003865000 nid=0xdd3d runnable "GC task thread#10 (ParallelGC)" prio=10 tid=0x0000000003864000 nid=0xdd3e runnable "GC task thread#11 (ParallelGC)" prio=10 tid=0x0000000003867800 nid=0xdd3f runnable "GC task thread#12 (ParallelGC)" prio=10 tid=0x0000000003873800 nid=0xdd40 runnable "GC task thread#13 (ParallelGC)" prio=10 tid=0x0000000003872800 nid=0xdd41 runnable "GC task thread#14 (ParallelGC)" prio=10 tid=0x0000000003872000 nid=0xdd42 runnable "GC task thread#15 (ParallelGC)" prio=10 tid=0x0000000003871800 nid=0xdd43 runnable "GC task thread#16 (ParallelGC)" prio=10 tid=0x0000000003871000 nid=0xdd44 runnable "GC task thread#17 (ParallelGC)" prio=10 tid=0x0000000003870000 nid=0xdd45 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000003992000 nid=0xdd4d waiting on condition JNI global references: 281