2019-02-13 03:34:35 Full thread dump OpenJDK 64-Bit Server VM (11.0.2+9 mixed mode): Threads class SMR info: _java_thread_list=0x00007fb3dcab3ef0, length=84, elements={ 0x00007fb498012800, 0x00007fb498e8e800, 0x00007fb498e92800, 0x00007fb498ea5800, 0x00007fb498ea7800, 0x00007fb498eaa000, 0x00007fb498eac000, 0x00007fb498f88800, 0x00007fb498fd9800, 0x00007fb499400000, 0x00007fb499418000, 0x00007fb499699000, 0x00007fb49969f000, 0x00007fb4996b7800, 0x00007fb4996e9000, 0x00007fb499825800, 0x00007fb499836800, 0x00007fb499838800, 0x00007fb49983e800, 0x00007fb3c8039800, 0x00007fb3c803b800, 0x00007fb3c8052800, 0x00007fb3c8055800, 0x00007fb3c8057000, 0x00007fb3c805b800, 0x00007fb3c805d000, 0x00007fb3c805f000, 0x00007fb3c8061000, 0x00007fb3c8063000, 0x00007fb498efd000, 0x00007fb4998f0000, 0x00007fb4998fc000, 0x00007fb4998eb000, 0x00007fb499914000, 0x00007fb49990f800, 0x00007fb499936000, 0x00007fb4999d0000, 0x00007fb4999d7000, 0x00007fb4999dd000, 0x00007fb4999de800, 0x00007fb4999f1000, 0x00007fb499a12800, 0x00007fb499a14800, 0x00007fb499a37000, 0x00007fb499a38800, 0x00007fb499a44000, 0x00007fb499a3b800, 0x00007fb499a5f000, 0x00007fb499a63000, 0x00007fb499a64000, 0x00007fb499a66000, 0x00007fb499a67800, 0x00007fb499a69800, 0x00007fb499a6b800, 0x00007fb499a6d800, 0x00007fb499ae8000, 0x00007fb499b18000, 0x00007fb499b1a000, 0x00007fb499b1c000, 0x00000000014e6000, 0x00007fa9d800a000, 0x00007fa9d800b800, 0x00007fb499b36000, 0x00007fb444001000, 0x00007fb499b3c800, 0x00007fb49998a000, 0x00007fb49998c000, 0x00007faa54001000, 0x00007fa9fc0a6000, 0x00007fa9fc124800, 0x00007fa9fc129000, 0x00007fa9fc12d800, 0x00007fa9f8055000, 0x00007fa9f8056800, 0x00007fa9f8061000, 0x00007fa9f8065800, 0x00007faa0002b800, 0x00007faa00035000, 0x00007faa0003b800, 0x00007faa00040800, 0x00007fa9f806b000, 0x00007fa9f806f800, 0x00007fa9f8075800, 0x00007fa9f807a000 } "main" #1 prio=5 os_prio=0 cpu=3678.79ms elapsed=324871.99s tid=0x00007fb498012800 nid=0x233c0 waiting on condition [0x00007fb49e63d000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006cb0bfb48> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.2/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(java.base@11.0.2/AbstractQueuedSynchronizer.java:1039) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(java.base@11.0.2/AbstractQueuedSynchronizer.java:1345) at java.util.concurrent.CountDownLatch.await(java.base@11.0.2/CountDownLatch.java:232) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:644) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=47.84ms elapsed=324871.90s tid=0x00007fb498e8e800 nid=0x233c9 waiting on condition [0x00007fb469efc000] java.lang.Thread.State: RUNNABLE at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.2/Native Method) at java.lang.ref.Reference.processPendingReferences(java.base@11.0.2/Reference.java:241) at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.2/Reference.java:213) "Finalizer" #3 daemon prio=8 os_prio=0 cpu=276.56ms elapsed=324871.90s tid=0x00007fb498e92800 nid=0x233ca in Object.wait() [0x00007fb4696fb000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.2/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x00000006c0001d18> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:176) at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.2/Finalizer.java:170) "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.33ms elapsed=324871.89s tid=0x00007fb498ea5800 nid=0x233ce runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 cpu=55253.71ms elapsed=324871.89s tid=0x00007fb498ea7800 nid=0x233cf waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "C1 CompilerThread0" #8 daemon prio=9 os_prio=0 cpu=6755.21ms elapsed=324871.89s tid=0x00007fb498eaa000 nid=0x233d0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "Sweeper thread" #9 daemon prio=9 os_prio=0 cpu=9934.66ms elapsed=324871.89s tid=0x00007fb498eac000 nid=0x233d1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Common-Cleaner" #10 daemon prio=8 os_prio=0 cpu=190.25ms elapsed=324871.84s tid=0x00007fb498f88800 nid=0x233d3 in Object.wait() [0x00007fb4492d3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.2/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.2/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x00000006c0002328> (a java.lang.ref.ReferenceQueue$Lock) at jdk.internal.ref.CleanerImpl.run(java.base@11.0.2/CleanerImpl.java:148) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) at jdk.internal.misc.InnocuousThread.run(java.base@11.0.2/InnocuousThread.java:134) "Service Thread" #11 daemon prio=9 os_prio=0 cpu=0.10ms elapsed=324871.81s tid=0x00007fb498fd9800 nid=0x233d5 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RMI TCP Accept-0" #13 daemon prio=5 os_prio=0 cpu=0.47ms elapsed=324871.32s tid=0x00007fb499400000 nid=0x233db runnable [0x00007fb42b7f0000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(java.base@11.0.2/Native Method) at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.2/AbstractPlainSocketImpl.java:458) at java.net.ServerSocket.implAccept(java.base@11.0.2/ServerSocket.java:551) at java.net.ServerSocket.accept(java.base@11.0.2/ServerSocket.java:519) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(java.rmi@11.0.2/TCPTransport.java:394) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(java.rmi@11.0.2/TCPTransport.java:366) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "RMI TCP Accept-9144" #14 daemon prio=5 os_prio=0 cpu=2.84ms elapsed=324871.31s tid=0x00007fb499418000 nid=0x233dc runnable [0x00007fb42adef000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(java.base@11.0.2/Native Method) at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.2/AbstractPlainSocketImpl.java:458) at java.net.ServerSocket.implAccept(java.base@11.0.2/ServerSocket.java:551) at java.net.ServerSocket.accept(java.base@11.0.2/ServerSocket.java:519) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(java.rmi@11.0.2/TCPTransport.java:394) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(java.rmi@11.0.2/TCPTransport.java:366) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "metrics-meter-tick-thread-1" #17 daemon prio=5 os_prio=0 cpu=3093790.80ms elapsed=324870.63s tid=0x00007fb499699000 nid=0x233e2 waiting on condition [0x00007fb429fee000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0003218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "metrics-meter-tick-thread-2" #18 daemon prio=5 os_prio=0 cpu=3032955.72ms elapsed=324870.63s tid=0x00007fb49969f000 nid=0x233e3 waiting on condition [0x00007fb4297ed000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0003218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "main-SendThread(mwkafka-zk-prod-05.tbd:2181)" #22 daemon prio=5 os_prio=0 cpu=4753.19ms elapsed=324870.13s tid=0x00007fb4996b7800 nid=0x233f6 runnable [0x00007fb413ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006c0009ed0> (a sun.nio.ch.Util$2) - locked <0x00000006c00099b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) "main-EventThread" #23 daemon prio=5 os_prio=0 cpu=204.16ms elapsed=324870.13s tid=0x00007fb4996e9000 nid=0x233f7 waiting on condition [0x00007fb410af7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0024588> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) "SensorExpiryThread" #24 daemon prio=5 os_prio=0 cpu=784.70ms elapsed=324869.83s tid=0x00007fb499825800 nid=0x233fa waiting on condition [0x00007fb428fec000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022968> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "ThrottledChannelReaper-Fetch" #25 prio=5 os_prio=0 cpu=6402.17ms elapsed=324869.82s tid=0x00007fb499836800 nid=0x233fb waiting on condition [0x00007fb3dbdfe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c00247f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ThrottledChannelReaper-Produce" #26 prio=5 os_prio=0 cpu=6350.33ms elapsed=324869.81s tid=0x00007fb499838800 nid=0x233fc waiting on condition [0x00007fb3db5fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022c68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ThrottledChannelReaper-Request" #27 prio=5 os_prio=0 cpu=6419.05ms elapsed=324869.81s tid=0x00007fb49983e800 nid=0x233fd waiting on condition [0x00007fb3dadfc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0024b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-scheduler-0" #29 daemon prio=5 os_prio=0 cpu=72020.87ms elapsed=324868.90s tid=0x00007fb3c8039800 nid=0x2340b waiting on condition [0x00007fb3c3ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-1" #30 daemon prio=5 os_prio=0 cpu=70074.74ms elapsed=324868.89s tid=0x00007fb3c803b800 nid=0x2340c waiting on condition [0x00007fb3c23fd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-2" #31 daemon prio=5 os_prio=0 cpu=73744.45ms elapsed=324868.87s tid=0x00007fb3c8052800 nid=0x2340d waiting on condition [0x00007fb3bbffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-3" #32 daemon prio=5 os_prio=0 cpu=71631.14ms elapsed=324868.85s tid=0x00007fb3c8055800 nid=0x2340e waiting on condition [0x00007fb3ba3fd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-4" #33 daemon prio=5 os_prio=0 cpu=73124.73ms elapsed=324868.84s tid=0x00007fb3c8057000 nid=0x2340f waiting on condition [0x00007fb3afffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-5" #34 daemon prio=5 os_prio=0 cpu=71417.01ms elapsed=324868.83s tid=0x00007fb3c805b800 nid=0x23410 waiting on condition [0x00007fb3ae3fd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-6" #35 daemon prio=5 os_prio=0 cpu=69592.60ms elapsed=324868.82s tid=0x00007fb3c805d000 nid=0x23411 waiting on condition [0x00007fb3a3ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-7" #36 daemon prio=5 os_prio=0 cpu=71769.53ms elapsed=324868.81s tid=0x00007fb3c805f000 nid=0x23412 waiting on condition [0x00007fb3a23fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-8" #37 daemon prio=5 os_prio=0 cpu=72052.56ms elapsed=324868.79s tid=0x00007fb3c8061000 nid=0x23413 waiting on condition [0x00007fb397ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-scheduler-9" #38 daemon prio=5 os_prio=0 cpu=71459.56ms elapsed=324868.78s tid=0x00007fb3c8063000 nid=0x23414 waiting on condition [0x00007fb3963fd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c0022cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-log-cleaner-thread-0" #39 prio=5 os_prio=0 cpu=21763.06ms elapsed=324863.66s tid=0x00007fb498efd000 nid=0x23443 waiting on condition [0x00007fb3da2fa000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1cf910> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:1369) at java.util.concurrent.CountDownLatch.await(java.base@11.0.2/CountDownLatch.java:278) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:302) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-kerberos-refresh-thread-kafka/mwkafka-prod-02.tbd@UNIX.DESHAW.COM" #40 daemon prio=5 os_prio=0 cpu=237.38ms elapsed=324863.45s tid=0x00007fb4998f0000 nid=0x23445 waiting on condition [0x00007faa64ffe000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(java.base@11.0.2/Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin.lambda$login$10(KerberosLogin.java:194) at org.apache.kafka.common.security.kerberos.KerberosLogin$$Lambda$213/0x0000000800437840.run(Unknown Source) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9092" #41 prio=5 os_prio=0 cpu=15198.00ms elapsed=324863.43s tid=0x00007fb4998fc000 nid=0x23446 runnable [0x00007faa5fffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006ca1c2218> (a sun.nio.ch.Util$2) - locked <0x00000006ca1c21c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at kafka.network.Acceptor.run(SocketServer.scala:390) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "ExpirationReaper-6-Produce" #42 prio=5 os_prio=0 cpu=27613.88ms elapsed=324863.40s tid=0x00007fb4998eb000 nid=0x23447 waiting on condition [0x00007faa5f7fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1cfb88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-6-Fetch" #43 prio=5 os_prio=0 cpu=47238.14ms elapsed=324863.40s tid=0x00007fb499914000 nid=0x23448 waiting on condition [0x00007faa5effc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c90c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-6-DeleteRecords" #44 prio=5 os_prio=0 cpu=27596.89ms elapsed=324863.40s tid=0x00007fb49990f800 nid=0x23449 waiting on condition [0x00007faa5e7fb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c2480> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "LogDirFailureHandler" #45 prio=5 os_prio=0 cpu=0.18ms elapsed=324863.40s tid=0x00007fb499936000 nid=0x2344a waiting on condition [0x00007faa5dffa000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006c000b018> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.2/ArrayBlockingQueue.java:417) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:56) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:200) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "controller-event-thread" #46 prio=5 os_prio=0 cpu=4.64ms elapsed=324863.15s tid=0x00007fb4999d0000 nid=0x2344d waiting on condition [0x00007faa5d7f9000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caa49410> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:80) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-6-topic" #47 prio=5 os_prio=0 cpu=27157.14ms elapsed=324863.14s tid=0x00007fb4999d7000 nid=0x2344e waiting on condition [0x00007faa5cff8000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caa425c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-6-Heartbeat" #48 prio=5 os_prio=0 cpu=26786.77ms elapsed=324863.14s tid=0x00007fb4999dd000 nid=0x2344f waiting on condition [0x00007faa37ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caa43758> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-6-Rebalance" #49 prio=5 os_prio=0 cpu=27360.13ms elapsed=324863.14s tid=0x00007fb4999de800 nid=0x23450 waiting on condition [0x00007faa377fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caa443c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.2/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "group-metadata-manager-0" #50 daemon prio=5 os_prio=0 cpu=6711.73ms elapsed=324863.12s tid=0x00007fb4999f1000 nid=0x23451 waiting on condition [0x00007faa36ffc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca9f16f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "transaction-log-manager-0" #53 daemon prio=5 os_prio=0 cpu=512.35ms elapsed=324863.09s tid=0x00007fb499a12800 nid=0x23452 waiting on condition [0x00007faa367fb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caa41950> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "TxnMarkerSenderThread-6" #52 prio=5 os_prio=0 cpu=1218.57ms elapsed=324863.09s tid=0x00007fb499a14800 nid=0x23453 runnable [0x00007faa35ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006ca9f19a0> (a sun.nio.ch.Util$2) - locked <0x00000006ca9f19b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "main-SendThread(mwkafka-zk-prod-04.dr:2181)" #54 daemon prio=5 os_prio=0 cpu=3758.83ms elapsed=324863.06s tid=0x00007fb499a37000 nid=0x23454 runnable [0x00007faa357f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006ca9f1dc8> (a sun.nio.ch.Util$2) - locked <0x00000006ca9f1d70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) "main-EventThread" #55 daemon prio=5 os_prio=0 cpu=5.09ms elapsed=324863.06s tid=0x00007fb499a38800 nid=0x23455 waiting on condition [0x00007faa34ff8000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca9f20f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) "/kafka-acl-changes-event-process-thread" #56 prio=5 os_prio=0 cpu=0.54ms elapsed=324862.98s tid=0x00007fb499a44000 nid=0x23456 waiting on condition [0x00007faa17ffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca9f2ee8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "/kafka-acl-extended-changes-event-process-thread" #57 prio=5 os_prio=0 cpu=39.94ms elapsed=324862.97s tid=0x00007fb499a3b800 nid=0x23457 waiting on condition [0x00007faa177fd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caa41c60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-request-handler-0" #58 daemon prio=5 os_prio=0 cpu=59480.04ms elapsed=324862.84s tid=0x00007fb499a5f000 nid=0x2345e waiting on condition [0x00007faa16ffc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-1" #59 daemon prio=5 os_prio=0 cpu=59738.92ms elapsed=324862.84s tid=0x00007fb499a63000 nid=0x2345f waiting on condition [0x00007faa167fb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-2" #60 daemon prio=5 os_prio=0 cpu=65092.66ms elapsed=324862.84s tid=0x00007fb499a64000 nid=0x23460 waiting on condition [0x00007faa15ffa000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-3" #61 daemon prio=5 os_prio=0 cpu=60344.65ms elapsed=324862.84s tid=0x00007fb499a66000 nid=0x23461 waiting on condition [0x00007faa157f9000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-4" #62 daemon prio=5 os_prio=0 cpu=60133.53ms elapsed=324862.84s tid=0x00007fb499a67800 nid=0x23462 waiting on condition [0x00007faa14ff8000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-5" #63 daemon prio=5 os_prio=0 cpu=59292.63ms elapsed=324862.84s tid=0x00007fb499a69800 nid=0x23463 waiting on condition [0x00007fa9f7ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-6" #64 daemon prio=5 os_prio=0 cpu=59661.82ms elapsed=324862.84s tid=0x00007fb499a6b800 nid=0x23464 waiting on condition [0x00007fa9f77fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-request-handler-7" #65 daemon prio=5 os_prio=0 cpu=59526.21ms elapsed=324862.84s tid=0x00007fb499a6d800 nid=0x23465 waiting on condition [0x00007fa9f6ffc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca1c9cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ArrayBlockingQueue.poll(java.base@11.0.2/ArrayBlockingQueue.java:432) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:54) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "AsynchronousOutputStream-1-metric_plugin.log" #66 daemon prio=5 os_prio=0 cpu=10.52ms elapsed=324862.76s tid=0x00007fb499ae8000 nid=0x23468 in Object.wait() [0x00007fa9f67fb000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.2/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.2/Object.java:328) at common.io.AsynchronousOutputStream$WriterRunnable.drainBuffer(AsynchronousOutputStream.java:124) - waiting to re-lock in wait() <0x00000006caabea10> (a common.io.AsynchronousOutputStream$CircularByteBuffer) at common.io.AsynchronousOutputStream$WriterRunnable.copy(AsynchronousOutputStream.java:98) at common.io.AsynchronousOutputStream$WriterRunnable.run(AsynchronousOutputStream.java:66) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "Sonar-Transport-Manager" #67 daemon prio=5 os_prio=0 cpu=2755.00ms elapsed=324862.73s tid=0x00007fb499b18000 nid=0x23469 waiting on condition [0x00007fa9f5ffa000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006caabecf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at common.util.SeparateThreadWorker.run(SeparateThreadWorker.java:211) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "Sonar-Transponder-Worker" #68 daemon prio=5 os_prio=0 cpu=25914.25ms elapsed=324862.73s tid=0x00007fb499b1a000 nid=0x2346a waiting on condition [0x00007fa9f57f9000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006cab35c18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at common.util.SeparateThreadWorker.run(SeparateThreadWorker.java:211) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "Sonar-Transponder-Scheduler-1" #69 daemon prio=5 os_prio=0 cpu=1644.26ms elapsed=324862.73s tid=0x00007fb499b1c000 nid=0x2346b waiting on condition [0x00007fa9f4ff8000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006cab35dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "Sonar-TransponderTaskManager-Worker-1" #70 daemon prio=5 os_prio=0 cpu=617.50ms elapsed=324862.73s tid=0x00000000014e6000 nid=0x2346c waiting on condition [0x00007fa9dfffe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006cab36058> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.2/ArrayBlockingQueue.java:417) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "StatsCollector-1" #72 daemon prio=5 os_prio=0 cpu=24649.14ms elapsed=324862.72s tid=0x00007fa9d800a000 nid=0x2346d waiting on condition [0x00007fa9df7fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006cab39bf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.2/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.2/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "AsynchronousOutputStream-2-metric_plugin.stats.log" #73 daemon prio=5 os_prio=0 cpu=1029.72ms elapsed=324862.72s tid=0x00007fa9d800b800 nid=0x2346e in Object.wait() [0x00007fa9deffc000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.2/Native Method) - waiting on at java.lang.Object.wait(java.base@11.0.2/Object.java:328) at common.io.AsynchronousOutputStream$WriterRunnable.drainBuffer(AsynchronousOutputStream.java:124) - waiting to re-lock in wait() <0x00000006cab31810> (a common.io.AsynchronousOutputStream$CircularByteBuffer) at common.io.AsynchronousOutputStream$WriterRunnable.copy(AsynchronousOutputStream.java:98) at common.io.AsynchronousOutputStream$WriterRunnable.run(AsynchronousOutputStream.java:66) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "/config/changes-event-process-thread" #74 prio=5 os_prio=0 cpu=0.40ms elapsed=324862.71s tid=0x00007fb499b36000 nid=0x2346f waiting on condition [0x00007fa9de7fb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006cab39ee8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "Attach Listener" #76 daemon prio=9 os_prio=0 cpu=1.82ms elapsed=324862.31s tid=0x00007fb444001000 nid=0x23485 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "kafka-network-thread-6-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #77 prio=5 os_prio=0 cpu=13533450.55ms elapsed=324860.90s tid=0x00007fb499b3c800 nid=0x2349d runnable [0x00007fa9dd7f5000] java.lang.Thread.State: RUNNABLE at java.util.HashMap$TreeNode.find(java.base@11.0.2/HashMap.java:1928) at java.util.HashMap$TreeNode.find(java.base@11.0.2/HashMap.java:1924) at java.util.HashMap$TreeNode.find(java.base@11.0.2/HashMap.java:1924) at java.util.HashMap$TreeNode.find(java.base@11.0.2/HashMap.java:1924) at java.util.HashMap$TreeNode.putTreeVal(java.base@11.0.2/HashMap.java:2043) at java.util.HashMap.putVal(java.base@11.0.2/HashMap.java:633) at java.util.HashMap.put(java.base@11.0.2/HashMap.java:607) at java.util.HashSet.add(java.base@11.0.2/HashSet.java:220) at javax.security.auth.Subject$ClassSet.populateSet(java.base@11.0.2/Subject.java:1518) at javax.security.auth.Subject$ClassSet.(java.base@11.0.2/Subject.java:1472) - locked <0x00000006ca1c9a80> (a java.util.Collections$SynchronizedSet) at javax.security.auth.Subject.getPrivateCredentials(java.base@11.0.2/Subject.java:764) at sun.security.jgss.GSSUtil$1.run(java.security.jgss@11.0.2/GSSUtil.java:336) at sun.security.jgss.GSSUtil$1.run(java.security.jgss@11.0.2/GSSUtil.java:328) at java.security.AccessController.doPrivileged(java.base@11.0.2/Native Method) at sun.security.jgss.GSSUtil.searchSubject(java.security.jgss@11.0.2/GSSUtil.java:328) at sun.security.jgss.wrapper.NativeGSSFactory.getCredFromSubject(java.security.jgss@11.0.2/NativeGSSFactory.java:53) at sun.security.jgss.wrapper.NativeGSSFactory.getCredentialElement(java.security.jgss@11.0.2/NativeGSSFactory.java:116) at sun.security.jgss.GSSManagerImpl.getCredentialElement(java.security.jgss@11.0.2/GSSManagerImpl.java:187) at sun.security.jgss.GSSCredentialImpl.add(java.security.jgss@11.0.2/GSSCredentialImpl.java:439) at sun.security.jgss.GSSCredentialImpl.(java.security.jgss@11.0.2/GSSCredentialImpl.java:74) at sun.security.jgss.GSSManagerImpl.createCredential(java.security.jgss@11.0.2/GSSManagerImpl.java:148) at com.sun.security.sasl.gsskerb.GssKrb5Server.(jdk.security.jgss@11.0.2/GssKrb5Server.java:108) at com.sun.security.sasl.gsskerb.FactoryImpl.createSaslServer(jdk.security.jgss@11.0.2/FactoryImpl.java:85) at javax.security.sasl.Sasl.createSaslServer(java.security.sasl@11.0.2/Sasl.java:537) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.lambda$createSaslKerberosServer$12(SaslServerAuthenticator.java:212) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator$$Lambda$236/0x00000008004f5440.run(Unknown Source) at java.security.AccessController.doPrivileged(java.base@11.0.2/Native Method) at javax.security.auth.Subject.doAs(java.base@11.0.2/Subject.java:423) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.createSaslKerberosServer(SaslServerAuthenticator.java:211) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.createSaslServer(SaslServerAuthenticator.java:164) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleKafkaRequest(SaslServerAuthenticator.java:450) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:248) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:132) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:532) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-network-thread-6-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #78 prio=5 os_prio=0 cpu=13526074.02ms elapsed=324860.90s tid=0x00007fb49998a000 nid=0x2349e waiting for monitor entry [0x00007fa9dcff8000] java.lang.Thread.State: BLOCKED (on object monitor) at java.util.Collections$SynchronizedCollection.add(java.base@11.0.2/Collections.java:2038) - waiting to lock <0x00000006ca1c9a80> (a java.util.Collections$SynchronizedSet) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.createSaslKerberosServer(SaslServerAuthenticator.java:204) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.createSaslServer(SaslServerAuthenticator.java:164) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleKafkaRequest(SaslServerAuthenticator.java:450) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:248) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:132) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:532) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "kafka-network-thread-6-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #79 prio=5 os_prio=0 cpu=13480566.67ms elapsed=324860.90s tid=0x00007fb49998c000 nid=0x2349f waiting for monitor entry [0x00007fa9d7ffb000] java.lang.Thread.State: BLOCKED (on object monitor) at javax.security.auth.Subject$ClassSet.(java.base@11.0.2/Subject.java:1472) - waiting to lock <0x00000006ca1c9a80> (a java.util.Collections$SynchronizedSet) at javax.security.auth.Subject.getPrivateCredentials(java.base@11.0.2/Subject.java:764) at sun.security.jgss.GSSUtil$1.run(java.security.jgss@11.0.2/GSSUtil.java:336) at sun.security.jgss.GSSUtil$1.run(java.security.jgss@11.0.2/GSSUtil.java:328) at java.security.AccessController.doPrivileged(java.base@11.0.2/Native Method) at sun.security.jgss.GSSUtil.searchSubject(java.security.jgss@11.0.2/GSSUtil.java:328) at sun.security.jgss.wrapper.NativeGSSFactory.getCredFromSubject(java.security.jgss@11.0.2/NativeGSSFactory.java:53) at sun.security.jgss.wrapper.NativeGSSFactory.getCredentialElement(java.security.jgss@11.0.2/NativeGSSFactory.java:116) at sun.security.jgss.GSSManagerImpl.getCredentialElement(java.security.jgss@11.0.2/GSSManagerImpl.java:187) at sun.security.jgss.GSSCredentialImpl.add(java.security.jgss@11.0.2/GSSCredentialImpl.java:439) at sun.security.jgss.GSSCredentialImpl.(java.security.jgss@11.0.2/GSSCredentialImpl.java:74) at sun.security.jgss.GSSManagerImpl.createCredential(java.security.jgss@11.0.2/GSSManagerImpl.java:148) at com.sun.security.sasl.gsskerb.GssKrb5Server.(jdk.security.jgss@11.0.2/GssKrb5Server.java:108) at com.sun.security.sasl.gsskerb.FactoryImpl.createSaslServer(jdk.security.jgss@11.0.2/FactoryImpl.java:85) at javax.security.sasl.Sasl.createSaslServer(java.security.sasl@11.0.2/Sasl.java:537) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.lambda$createSaslKerberosServer$12(SaslServerAuthenticator.java:212) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator$$Lambda$236/0x00000008004f5440.run(Unknown Source) at java.security.AccessController.doPrivileged(java.base@11.0.2/Native Method) at javax.security.auth.Subject.doAs(java.base@11.0.2/Subject.java:423) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.createSaslKerberosServer(SaslServerAuthenticator.java:211) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.createSaslServer(SaslServerAuthenticator.java:164) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleKafkaRequest(SaslServerAuthenticator.java:450) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:248) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:132) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:532) at org.apache.kafka.common.network.Selector.poll(Selector.java:467) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "executor-Fetch" #80 prio=5 os_prio=0 cpu=140585.76ms elapsed=324841.46s tid=0x00007faa54001000 nid=0x235c8 waiting on condition [0x00007fa9d77fd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ca8eea30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.2/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.2/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.2/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.2/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.2/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.2/Thread.java:834) "ReplicaFetcherThread-0-1" #81 prio=5 os_prio=0 cpu=195571.98ms elapsed=324823.55s tid=0x00007fa9fc0a6000 nid=0x23651 runnable [0x00007fa9d6ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8afc0> (a sun.nio.ch.Util$2) - locked <0x00000006cec8af68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-2-1" #82 prio=5 os_prio=0 cpu=261326.40ms elapsed=324823.53s tid=0x00007fa9fc124800 nid=0x23652 runnable [0x00007fa9d67fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8b828> (a sun.nio.ch.Util$2) - locked <0x00000006cec8b7d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-1-1" #83 prio=5 os_prio=0 cpu=184678.89ms elapsed=324823.52s tid=0x00007fa9fc129000 nid=0x23653 runnable [0x00007fa9d5ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec882b0> (a sun.nio.ch.Util$2) - locked <0x00000006cec88258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-3-1" #84 prio=5 os_prio=0 cpu=265054.53ms elapsed=324823.51s tid=0x00007fa9fc12d800 nid=0x23654 runnable [0x00007fa9d57f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8d350> (a sun.nio.ch.Util$2) - locked <0x00000006cec8d2f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-0-2" #85 prio=5 os_prio=0 cpu=258979.03ms elapsed=324823.27s tid=0x00007fa9f8055000 nid=0x23656 runnable [0x00007fa9d4ff8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8e610> (a sun.nio.ch.Util$2) - locked <0x00000006cec8e5b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-2-2" #86 prio=5 os_prio=0 cpu=268815.31ms elapsed=324823.25s tid=0x00007fa9f8056800 nid=0x23657 runnable [0x00007fa9d47f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8c2b0> (a sun.nio.ch.Util$2) - locked <0x00000006cec8c258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-3-2" #87 prio=5 os_prio=0 cpu=274133.63ms elapsed=324823.24s tid=0x00007fa9f8061000 nid=0x23658 runnable [0x00007fa9d3ff6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec88610> (a sun.nio.ch.Util$2) - locked <0x00000006cec885b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-1-2" #88 prio=5 os_prio=0 cpu=300890.05ms elapsed=324823.23s tid=0x00007fa9f8065800 nid=0x23659 runnable [0x00007fa9d37f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8b308> (a sun.nio.ch.Util$2) - locked <0x00000006cec8b2b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-0-3" #89 prio=5 os_prio=0 cpu=256813.78ms elapsed=324822.99s tid=0x00007faa0002b800 nid=0x2365b runnable [0x00007fa9d2ff4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8bb70> (a sun.nio.ch.Util$2) - locked <0x00000006cec8bb18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-2-3" #90 prio=5 os_prio=0 cpu=209045.01ms elapsed=324822.98s tid=0x00007faa00035000 nid=0x2365c runnable [0x00007fa9d27f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8c5f8> (a sun.nio.ch.Util$2) - locked <0x00000006cec8c5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-3-3" #91 prio=5 os_prio=0 cpu=259424.00ms elapsed=324822.97s tid=0x00007faa0003b800 nid=0x2365e runnable [0x00007fa9d1ff2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec89c68> (a sun.nio.ch.Util$2) - locked <0x00000006cec89c10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-1-3" #92 prio=5 os_prio=0 cpu=255187.88ms elapsed=324822.96s tid=0x00007faa00040800 nid=0x2365f waiting on condition [0x00007fa9d17f1000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.2/Native Method) - parking to wait for <0x00000006ced813d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.2/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.2/AbstractQueuedSynchronizer.java:2211) at kafka.server.AbstractFetcherThread$$anonfun$kafka$server$AbstractFetcherThread$$processFetchRequest$1.apply$mcZ$sp(AbstractFetcherThread.scala:251) at kafka.server.AbstractFetcherThread$$anonfun$kafka$server$AbstractFetcherThread$$processFetchRequest$1.apply(AbstractFetcherThread.scala:246) at kafka.server.AbstractFetcherThread$$anonfun$kafka$server$AbstractFetcherThread$$processFetchRequest$1.apply(AbstractFetcherThread.scala:246) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:246) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-2-4" #93 prio=5 os_prio=0 cpu=298397.70ms elapsed=324822.80s tid=0x00007fa9f806b000 nid=0x23661 runnable [0x00007fa9d0ff0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec89fb0> (a sun.nio.ch.Util$2) - locked <0x00000006cec89f58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-0-4" #94 prio=5 os_prio=0 cpu=309916.41ms elapsed=324822.79s tid=0x00007fa9f806f800 nid=0x23662 runnable [0x00007fa9d07ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec88ba0> (a sun.nio.ch.Util$2) - locked <0x00000006cec88b48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-1-4" #95 prio=5 os_prio=0 cpu=336229.51ms elapsed=324822.78s tid=0x00007fa9f8075800 nid=0x23663 runnable [0x00007fa9cffee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec88ee8> (a sun.nio.ch.Util$2) - locked <0x00000006cec88e90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-3-4" #96 prio=5 os_prio=0 cpu=320856.71ms elapsed=324822.78s tid=0x00007fa9f807a000 nid=0x23664 runnable [0x00007fa9cf7ed000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.2/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.2/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.2/SelectorImpl.java:124) - locked <0x00000006cec8e958> (a sun.nio.ch.Util$2) - locked <0x00000006cec8e900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.2/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:521) at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:73) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:92) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "VM Thread" os_prio=0 cpu=37848.30ms elapsed=324871.91s tid=0x00007fb498e86800 nid=0x233c8 runnable "GC Thread#0" os_prio=0 cpu=19329.22ms elapsed=324871.99s tid=0x00007fb49803a800 nid=0x233c1 runnable "GC Thread#1" os_prio=0 cpu=19114.12ms elapsed=324870.51s tid=0x00007fb45c001000 nid=0x233e7 runnable "GC Thread#2" os_prio=0 cpu=19065.60ms elapsed=324870.51s tid=0x00007fb45c002800 nid=0x233e8 runnable "GC Thread#3" os_prio=0 cpu=19205.76ms elapsed=324870.51s tid=0x00007fb45c004000 nid=0x233e9 runnable "GC Thread#4" os_prio=0 cpu=19312.68ms elapsed=324870.51s tid=0x00007fb45c006000 nid=0x233ea runnable "GC Thread#5" os_prio=0 cpu=19207.24ms elapsed=324870.51s tid=0x00007fb45c008000 nid=0x233eb runnable "GC Thread#6" os_prio=0 cpu=19170.27ms elapsed=324870.51s tid=0x00007fb45c009800 nid=0x233ec runnable "GC Thread#7" os_prio=0 cpu=19135.70ms elapsed=324870.51s tid=0x00007fb45c00b800 nid=0x233ed runnable "GC Thread#8" os_prio=0 cpu=19241.96ms elapsed=324870.51s tid=0x00007fb45c00d800 nid=0x233ee runnable "G1 Main Marker" os_prio=0 cpu=1.73ms elapsed=324871.99s tid=0x00007fb498061800 nid=0x233c2 runnable "G1 Conc#0" os_prio=0 cpu=197.26ms elapsed=324871.99s tid=0x00007fb498063800 nid=0x233c3 runnable "G1 Conc#1" os_prio=0 cpu=227.24ms elapsed=324870.47s tid=0x00007fb46c001000 nid=0x233ef runnable "G1 Refine#0" os_prio=0 cpu=17873.77ms elapsed=324871.94s tid=0x00007fb498e22800 nid=0x233c6 runnable "G1 Refine#1" os_prio=0 cpu=34.86ms elapsed=324863.29s tid=0x00007fb460001000 nid=0x2344b runnable "G1 Refine#2" os_prio=0 cpu=14.05ms elapsed=324863.29s tid=0x00007faa40001000 nid=0x2344c runnable "G1 Refine#3" os_prio=0 cpu=3.39ms elapsed=324860.52s tid=0x00007faa44001000 nid=0x234d2 runnable "G1 Refine#4" os_prio=0 cpu=0.24ms elapsed=324857.84s tid=0x00007faa0001b800 nid=0x23520 runnable "G1 Young RemSet Sampling" os_prio=0 cpu=235343.61ms elapsed=324871.94s tid=0x00007fb498e24800 nid=0x233c7 runnable "VM Periodic Task Thread" os_prio=0 cpu=81295.31ms elapsed=324871.28s tid=0x00007fb499423800 nid=0x233de waiting on condition JNI global refs: 33, weak refs: 0