2017-04-28 10:23:03 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.91-b14 mixed mode): "Attach Listener" #83 daemon prio=9 os_prio=0 tid=0x00007fe004001000 nid=0x6cd4 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RMI Scheduler(0)" #79 daemon prio=5 os_prio=0 tid=0x00007fde88005000 nid=0xc88a waiting on condition [0x00007fdf8a6fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004002ce0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "executor-Fetch" #77 prio=5 os_prio=0 tid=0x00007fdf00001000 nid=0xeefb waiting on condition [0x00007fdf6b6fb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400666a80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ReplicaFetcherThread-0-72" #76 prio=5 os_prio=0 tid=0x00007fdebc119800 nid=0xeefa runnable [0x00007fdf78aef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000408e32bb8> (a sun.nio.ch.Util$2) - locked <0x0000000408e32ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000408e329b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:135) at kafka.utils.NetworkClientBlockingOps$.pollContinuously$extension(NetworkClientBlockingOps.scala:142) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:249) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:234) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-9" #75 daemon prio=5 os_prio=0 tid=0x00007fdef8005000 nid=0xea2a waiting on condition [0x00007fdf6bbfc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-8" #74 daemon prio=5 os_prio=0 tid=0x00007fdeac067800 nid=0xea29 waiting on condition [0x00007fdf6bcfd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ReplicaFetcherThread-0-73" #73 prio=5 os_prio=0 tid=0x00007fdeac05f800 nid=0xea28 runnable [0x00007fdf6bdfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000408cc8c18> (a sun.nio.ch.Util$2) - locked <0x0000000408cc8c08> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000408cc8bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:349) at kafka.utils.NetworkClientBlockingOps$.recursivePoll$1(NetworkClientBlockingOps.scala:135) at kafka.utils.NetworkClientBlockingOps$.pollContinuously$extension(NetworkClientBlockingOps.scala:142) at kafka.utils.NetworkClientBlockingOps$.blockingSendAndReceive$extension(NetworkClientBlockingOps.scala:108) at kafka.server.ReplicaFetcherThread.sendRequest(ReplicaFetcherThread.scala:249) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:234) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:118) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-request-handler-14" #72 daemon prio=5 os_prio=0 tid=0x00007fe14129d800 nid=0xea27 waiting on condition [0x00007fdf42bbd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-13" #71 daemon prio=5 os_prio=0 tid=0x00007fe14129b800 nid=0xea26 waiting on condition [0x00007fdf42cbe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-12" #70 daemon prio=5 os_prio=0 tid=0x00007fe14129a000 nid=0xea25 waiting on condition [0x00007fdf42dbf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-11" #69 daemon prio=5 os_prio=0 tid=0x00007fe141298000 nid=0xea24 waiting on condition [0x00007fdf42ec0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-10" #68 daemon prio=5 os_prio=0 tid=0x00007fe141296000 nid=0xea23 waiting on condition [0x00007fdf42fc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-9" #67 daemon prio=5 os_prio=0 tid=0x00007fe141294800 nid=0xea22 waiting on condition [0x00007fdf430c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-8" #66 daemon prio=5 os_prio=0 tid=0x00007fe141292800 nid=0xea21 waiting on condition [0x00007fdf431c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-7" #65 daemon prio=5 os_prio=0 tid=0x00007fe141291000 nid=0xea20 waiting on condition [0x00007fdf45bbb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-6" #64 daemon prio=5 os_prio=0 tid=0x00007fe14128f000 nid=0xea1f waiting on condition [0x00007fdf45cbc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-5" #63 daemon prio=5 os_prio=0 tid=0x00007fe14128d800 nid=0xea1e waiting on condition [0x00007fdf45dbd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-4" #62 daemon prio=5 os_prio=0 tid=0x00007fe14128b800 nid=0xea1d waiting on condition [0x00007fdf45ebe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-3" #61 daemon prio=5 os_prio=0 tid=0x00007fe14128a000 nid=0xea1c waiting on condition [0x00007fdf45fbf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-2" #60 daemon prio=5 os_prio=0 tid=0x00007fe141288000 nid=0xea1b waiting on condition [0x00007fdf460c0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-1" #59 daemon prio=5 os_prio=0 tid=0x00007fe141285000 nid=0xea1a waiting on condition [0x00007fdf461c1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "kafka-request-handler-0" #58 daemon prio=5 os_prio=0 tid=0x00007fe141283800 nid=0xea19 waiting on condition [0x00007fdf462c2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004006b6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:237) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:50) at java.lang.Thread.run(Thread.java:745) "group-metadata-manager-0" #57 daemon prio=5 os_prio=0 tid=0x00007fe141236800 nid=0xea18 waiting on condition [0x00007fdf463c3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067d500> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-74" #56 prio=5 os_prio=0 tid=0x00007fe14122c000 nid=0xea17 waiting on condition [0x00007fdf479bd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067d768> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-74" #55 prio=5 os_prio=0 tid=0x00007fe14122b000 nid=0xea16 waiting on condition [0x00007fdf47abe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400670658> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-74" #54 prio=5 os_prio=0 tid=0x00007fe141228000 nid=0xea15 waiting on condition [0x00007fdf47bbf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067b5b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-7" #53 daemon prio=5 os_prio=0 tid=0x00007fdf2c00a000 nid=0xea14 waiting on condition [0x00007fdf47cc0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-6" #52 daemon prio=5 os_prio=0 tid=0x00007fdf28002000 nid=0xea13 waiting on condition [0x00007fdf47fc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-5" #51 daemon prio=5 os_prio=0 tid=0x00007fe1410d6000 nid=0xea12 waiting on condition [0x00007fdf480c2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-4" #50 daemon prio=5 os_prio=0 tid=0x00007fe1410d2000 nid=0xea11 waiting on condition [0x00007fdf481c3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-2" #49 daemon prio=5 os_prio=0 tid=0x00007fe1410cf800 nid=0xea10 waiting on condition [0x00007fdf6d314000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067feb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ExpirationReaper-74" #48 prio=5 os_prio=0 tid=0x00007fe1410ce000 nid=0xea0f waiting on condition [0x00007fdf6d415000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067e2e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ExpirationReaper-74" #47 prio=5 os_prio=0 tid=0x00007fe1410cd000 nid=0xea0e waiting on condition [0x00007fdf6d516000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067b9b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:354) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:378) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #46 prio=5 os_prio=0 tid=0x00007fe1413da000 nid=0xea0d runnable [0x00007fdf6d617000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040067da38> (a sun.nio.ch.Util$2) - locked <0x000000040067da28> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040067d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:274) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-74-ListenerName(PLAINTEXT)-PLAINTEXT-4" #45 prio=5 os_prio=0 tid=0x00007fe1413d8800 nid=0xea0c runnable [0x00007fdf6d718000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000400680198> (a sun.nio.ch.Util$2) - locked <0x0000000400680188> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400680140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-74-ListenerName(PLAINTEXT)-PLAINTEXT-3" #44 prio=5 os_prio=0 tid=0x00007fe141119000 nid=0xea0b runnable [0x00007fdf6d819000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000408ca0258> (a sun.nio.ch.Util$2) - locked <0x0000000408ca8510> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000408cacf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-74-ListenerName(PLAINTEXT)-PLAINTEXT-2" #43 prio=5 os_prio=0 tid=0x00007fe141117000 nid=0xea0a runnable [0x00007fdf6d91a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040067bb08> (a sun.nio.ch.Util$2) - locked <0x000000040067baf8> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040067bab0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-74-ListenerName(PLAINTEXT)-PLAINTEXT-1" #42 prio=5 os_prio=0 tid=0x00007fe141115800 nid=0xea09 runnable [0x00007fdf6da1b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000400688ad8> (a sun.nio.ch.Util$2) - locked <0x0000000400688ac8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400688a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-network-thread-74-ListenerName(PLAINTEXT)-PLAINTEXT-0" #41 prio=5 os_prio=0 tid=0x00007fe141fbb000 nid=0xea08 runnable [0x00007fdf72dca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x000000040067e5a8> (a sun.nio.ch.Util$2) - locked <0x000000040067e598> (a java.util.Collections$UnmodifiableSet) - locked <0x000000040067e540> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:489) at org.apache.kafka.common.network.Selector.poll(Selector.java:298) at kafka.network.Processor.poll(SocketServer.scala:494) at kafka.network.Processor.run(SocketServer.scala:432) at java.lang.Thread.run(Thread.java:745) "kafka-log-cleaner-thread-0" #36 prio=5 os_prio=0 tid=0x00007fe14137d800 nid=0xea07 waiting on condition [0x00007fdf72ecb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400688d50> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:261) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:220) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "kafka-scheduler-3" #40 daemon prio=5 os_prio=0 tid=0x00007fe141376800 nid=0xea06 waiting on condition [0x00007fdf72fcc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-2" #39 daemon prio=5 os_prio=0 tid=0x00007fe1403b4800 nid=0xea05 waiting on condition [0x00007fdf730cd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-1" #38 daemon prio=5 os_prio=0 tid=0x00007fe1403b2800 nid=0xea04 waiting on condition [0x00007fdf731ce000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "kafka-scheduler-0" #37 daemon prio=5 os_prio=0 tid=0x00007fe1403b0800 nid=0xea03 waiting on condition [0x00007fdf732cf000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400154a70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "metrics-meter-tick-thread-1" #35 daemon prio=5 os_prio=0 tid=0x00007fe14098b800 nid=0xea02 waiting on condition [0x00007fe0104eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000040067feb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "ThrottledRequestReaper-Produce" #33 prio=5 os_prio=0 tid=0x00007fe1422fb800 nid=0xdb46 waiting on condition [0x00007fe0107ec000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400311ec0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:158) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "ThrottledRequestReaper-Fetch" #32 prio=5 os_prio=0 tid=0x00007fe1422f9800 nid=0xdb45 waiting on condition [0x00007fe0108ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400393640> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:158) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:63) "SensorExpiryThread" #31 daemon prio=5 os_prio=0 tid=0x00007fe1422f2800 nid=0xdb44 waiting on condition [0x00007fe0109ee000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400312080> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007fe14225b800 nid=0xdb43 waiting on condition [0x00007fe010cef000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400001238> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) "main-SendThread(10.52.208.72:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007fe142263800 nid=0xdb42 runnable [0x00007fe010df0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004000014d0> (a sun.nio.ch.Util$2) - locked <0x00000004000014e0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000400001488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) "ZkClient-EventThread-28-10.52.208.72:2181,10.52.208.73:2181,10.52.208.74:2181" #28 daemon prio=5 os_prio=0 tid=0x00007fe142252000 nid=0xdb41 waiting on condition [0x00007fe010ef1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400001770> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:68) "RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007fe1406f1000 nid=0xdb3f runnable [0x00007fe0116f3000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-9999" #24 daemon prio=5 os_prio=0 tid=0x00007fe1406d5000 nid=0xdb3e runnable [0x00007fe0117f4000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007fe1406c0000 nid=0xdb3d runnable [0x00007fe0118f5000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:745) "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007fe1403a3000 nid=0xdb39 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007fe14039f800 nid=0xdb38 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007fe14039d800 nid=0xdb37 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007fe14039b800 nid=0xdb36 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007fe140399000 nid=0xdb35 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007fe140397800 nid=0xdb34 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007fe140395000 nid=0xdb33 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007fe140392800 nid=0xdb32 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007fe140390800 nid=0xdb31 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007fe14038e800 nid=0xdb30 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007fe14038c000 nid=0xdb2f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007fe14038a000 nid=0xdb2e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007fe14037f800 nid=0xdb2d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007fe14037d800 nid=0xdb2c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007fe14037b800 nid=0xdb2b waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007fe140379000 nid=0xdb2a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007fe140377000 nid=0xdb29 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007fe140375800 nid=0xdb28 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fe140342800 nid=0xdb27 in Object.wait() [0x00007fe0251a8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000004000144a8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fe14033e000 nid=0xdb26 in Object.wait() [0x00007fe0252a9000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x0000000400014498> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) "main" #1 prio=5 os_prio=0 tid=0x00007fe14000d000 nid=0xdaee waiting on condition [0x00007fe1454a8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000400680458> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:624) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:70) at kafka.Kafka$.main(Kafka.scala:68) at kafka.Kafka.main(Kafka.scala) "VM Thread" os_prio=0 tid=0x00007fe140336000 nid=0xdb25 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140025000 nid=0xdaef runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140027000 nid=0xdaf0 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140029000 nid=0xdaf1 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14002a800 nid=0xdaf2 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14002c800 nid=0xdaf3 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14002e800 nid=0xdaf4 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140030000 nid=0xdaf5 runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140032000 nid=0xdaf6 runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140034000 nid=0xdaf7 runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140035800 nid=0xdaf8 runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140037800 nid=0xdaf9 runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140039000 nid=0xdafa runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14003b000 nid=0xdafb runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14003d000 nid=0xdafc runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14003e800 nid=0xdafd runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140040800 nid=0xdafe runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140042800 nid=0xdaff runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140044000 nid=0xdb00 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140046000 nid=0xdb01 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140047800 nid=0xdb02 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007fe140049800 nid=0xdb03 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14004b800 nid=0xdb04 runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007fe14004d000 nid=0xdb05 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007fe14008f000 nid=0xdb1e runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fe140090800 nid=0xdb1f runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fe140092800 nid=0xdb20 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fe140094800 nid=0xdb21 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fe140096000 nid=0xdb22 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fe140098000 nid=0xdb23 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fe14009a000 nid=0xdb24 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007fe14007b000 nid=0xdb1d runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007fe140079800 nid=0xdb1c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007fe140077800 nid=0xdb1b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007fe140075800 nid=0xdb1a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007fe140074000 nid=0xdb19 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007fe140072000 nid=0xdb18 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007fe140070000 nid=0xdb17 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007fe14006e800 nid=0xdb16 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007fe14006c800 nid=0xdb15 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007fe14006a800 nid=0xdb14 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007fe140069000 nid=0xdb13 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007fe140067000 nid=0xdb12 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007fe140065000 nid=0xdb11 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007fe140063000 nid=0xdb10 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007fe140061800 nid=0xdb0f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007fe14005f800 nid=0xdb0e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007fe14005d800 nid=0xdb0d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007fe14005c000 nid=0xdb0c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007fe14005a000 nid=0xdb0b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007fe140058000 nid=0xdb0a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007fe140056800 nid=0xdb09 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007fe140054800 nid=0xdb08 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007fe140052800 nid=0xdb07 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007fe140051000 nid=0xdb06 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007fe1406f4000 nid=0xdb40 waiting on condition JNI global references: 2441