2019-01-09 08:07:11 Full thread dump Java HotSpot(TM) 64-Bit Server VM (11.0.1+13-LTS mixed mode): Threads class SMR info: _java_thread_list=0x00007f5458002bd0, length=76, elements={ 0x00007f54987f2000, 0x00007f54987f6800, 0x00007f5498809000, 0x00007f549880b000, 0x00007f549880d000, 0x00007f549880f800, 0x00007f54988a2000, 0x00007f54988f4800, 0x00007f5498b14000, 0x00007f5498b1b000, 0x00007f5498c68800, 0x00007f5498c7c800, 0x00007f5498dc8000, 0x00007f5498dc4800, 0x00007f5498f83000, 0x00007f5498f7f000, 0x00007f5498fa5000, 0x00007f5498fa2800, 0x00007f5498fa9800, 0x00007f5498fae800, 0x00007f54991d4800, 0x00007f54991f4000, 0x00007f54991f6000, 0x00007f54991f8800, 0x00007f53d4052000, 0x00007f53d4053000, 0x00007f53d4055800, 0x00007f53d4058800, 0x00007f53d4057800, 0x00007f53d405c800, 0x00007f53d405a800, 0x00007f53d405f000, 0x00007f53d4065000, 0x00007f53d4066000, 0x00007f54988ec800, 0x00007f549926c000, 0x00007f549928c800, 0x00007f5499285800, 0x00007f5499287800, 0x00007f54992ad800, 0x00007f54992fb000, 0x00007f5499302000, 0x00007f5499308800, 0x00007f5499309800, 0x00007f5499316000, 0x00007f549933d800, 0x00007f549933f800, 0x00007f5499354800, 0x00007f5499356000, 0x00007f5499359000, 0x00007f549935b000, 0x00007f549935c800, 0x00007f549935e800, 0x00007f5499360800, 0x00007f5499362000, 0x00007f549936f800, 0x00007f5499371000, 0x00007f5499387000, 0x00007f5499389000, 0x00007f5498011800, 0x00007f541402d000, 0x00007f5414034000, 0x00007f541403c800, 0x00007f5414041800, 0x00007f53c8004000, 0x00007f53c0006800, 0x00007f53e0002800, 0x00007f542400a000, 0x00007f5424010000, 0x00007f53e4001000, 0x00007f5424019000, 0x00007f5424006000, 0x00007f542401b800, 0x00007f5424028000, 0x00007f5424029800, 0x00007f5458001800 } "Reference Handler" #2 daemon prio=10 os_prio=0 cpu=137.13ms elapsed=84626.05s tid=0x00007f54987f2000 nid=0x3533 waiting on condition [0x00007f5474ef5000] java.lang.Thread.State: RUNNABLE at java.lang.ref.Reference.waitForReferencePendingList(java.base@11.0.1/Native Method) at java.lang.ref.Reference.processPendingReferences(java.base@11.0.1/Reference.java:241) at java.lang.ref.Reference$ReferenceHandler.run(java.base@11.0.1/Reference.java:213) "Finalizer" #3 daemon prio=8 os_prio=0 cpu=0.49ms elapsed=84626.04s tid=0x00007f54987f6800 nid=0x3534 in Object.wait() [0x00007f5474df4000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.1/Native Method) - waiting on <0x0000000088400bf0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.1/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x0000000088400bf0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(java.base@11.0.1/ReferenceQueue.java:176) at java.lang.ref.Finalizer$FinalizerThread.run(java.base@11.0.1/Finalizer.java:170) "Signal Dispatcher" #4 daemon prio=9 os_prio=0 cpu=0.53ms elapsed=84626.03s tid=0x00007f5498809000 nid=0x3535 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #5 daemon prio=9 os_prio=0 cpu=184293.79ms elapsed=84626.03s tid=0x00007f549880b000 nid=0x3536 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "C1 CompilerThread0" #7 daemon prio=9 os_prio=0 cpu=8136.53ms elapsed=84626.03s tid=0x00007f549880d000 nid=0x3537 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE No compile task "Sweeper thread" #8 daemon prio=9 os_prio=0 cpu=5633.42ms elapsed=84626.03s tid=0x00007f549880f800 nid=0x3538 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Common-Cleaner" #9 daemon prio=8 os_prio=0 cpu=54.81ms elapsed=84625.98s tid=0x00007f54988a2000 nid=0x3539 in Object.wait() [0x00007f54748ef000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(java.base@11.0.1/Native Method) - waiting on at java.lang.ref.ReferenceQueue.remove(java.base@11.0.1/ReferenceQueue.java:155) - waiting to re-lock in wait() <0x00000000884010d0> (a java.lang.ref.ReferenceQueue$Lock) at jdk.internal.ref.CleanerImpl.run(java.base@11.0.1/CleanerImpl.java:148) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) at jdk.internal.misc.InnocuousThread.run(java.base@11.0.1/InnocuousThread.java:134) "Service Thread" #10 daemon prio=9 os_prio=0 cpu=0.16ms elapsed=84625.95s tid=0x00007f54988f4800 nid=0x353a runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RMI TCP Accept-0" #12 daemon prio=5 os_prio=0 cpu=0.98ms elapsed=84625.58s tid=0x00007f5498b14000 nid=0x353f runnable [0x00007f54741c4000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(java.base@11.0.1/Native Method) at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.1/AbstractPlainSocketImpl.java:458) at java.net.ServerSocket.implAccept(java.base@11.0.1/ServerSocket.java:551) at java.net.ServerSocket.accept(java.base@11.0.1/ServerSocket.java:519) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(java.rmi@11.0.1/TCPTransport.java:394) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(java.rmi@11.0.1/TCPTransport.java:366) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "RMI TCP Accept-36140" #13 daemon prio=5 os_prio=0 cpu=0.19ms elapsed=84625.57s tid=0x00007f5498b1b000 nid=0x3540 runnable [0x00007f544d75b000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(java.base@11.0.1/Native Method) at java.net.AbstractPlainSocketImpl.accept(java.base@11.0.1/AbstractPlainSocketImpl.java:458) at java.net.ServerSocket.implAccept(java.base@11.0.1/ServerSocket.java:551) at java.net.ServerSocket.accept(java.base@11.0.1/ServerSocket.java:519) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(java.rmi@11.0.1/TCPTransport.java:394) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(java.rmi@11.0.1/TCPTransport.java:366) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-1" #14 daemon prio=5 os_prio=0 cpu=47.32ms elapsed=84625.22s tid=0x00007f5498c68800 nid=0x3542 waiting on condition [0x00007f544d159000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "AsyncAppender-Worker-appender" #15 daemon prio=5 os_prio=0 cpu=1376.10ms elapsed=84625.20s tid=0x00007f5498c7c800 nid=0x3543 waiting on condition [0x00007f544d058000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088403918> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.1/ArrayBlockingQueue.java:417) at ch.qos.logback.core.AsyncAppenderBase$Worker.run(AsyncAppenderBase.java:289) "pool-2-thread-1" #16 prio=5 os_prio=0 cpu=10569.99ms elapsed=84624.80s tid=0x00007f5498dc8000 nid=0x354a waiting on condition [0x00007f544c551000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088834178> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.1/ArrayBlockingQueue.java:417) at com.jpam.am.hawkeye.transport.udp.client.AsynchronizedUdpMessageSender.startAsyncSender(AsynchronizedUdpMessageSender.java:57) at com.jpam.am.hawkeye.transport.udp.client.AsynchronizedUdpMessageSender$$Lambda$123/0x00000001002a8440.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "Thread-1" #17 daemon prio=5 os_prio=0 cpu=24292.60ms elapsed=84624.79s tid=0x00007f5498dc4800 nid=0x354b waiting on condition [0x00007f544c450000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x000000008884a348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "main-SendThread(vsia2p4090.svr.apac.jpmchase.net:36100)" #19 daemon prio=5 os_prio=0 cpu=5480.68ms elapsed=84624.47s tid=0x00007f5498f83000 nid=0x354c runnable [0x00007f544c14f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x000000008884a618> (a sun.nio.ch.Util$2) - locked <0x000000008884a628> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1214) "main-EventThread" #20 daemon prio=5 os_prio=0 cpu=0.52ms elapsed=84624.47s tid=0x00007f5498f7f000 nid=0x354d waiting on condition [0x00007f53ffbfe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088834500> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.1/LinkedBlockingQueue.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:496) "metrics-meter-tick-thread-1" #21 daemon prio=5 os_prio=0 cpu=77077.01ms elapsed=84624.42s tid=0x00007f5498fa5000 nid=0x354e waiting on condition [0x00007f53ffafd000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088876658> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "metrics-meter-tick-thread-2" #22 daemon prio=5 os_prio=0 cpu=76856.55ms elapsed=84624.42s tid=0x00007f5498fa2800 nid=0x354f waiting on condition [0x00007f53ff9fc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088876658> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "main-SendThread(vsia2p4089.svr.apac.jpmchase.net:36100)" #23 daemon prio=5 os_prio=0 cpu=4964.18ms elapsed=84624.41s tid=0x00007f5498fa9800 nid=0x3550 runnable [0x00007f53ff8fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000888768c0> (a sun.nio.ch.Util$2) - locked <0x00000000888768d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1214) "main-EventThread" #24 daemon prio=5 os_prio=0 cpu=40.96ms elapsed=84624.41s tid=0x00007f5498fae800 nid=0x3551 waiting on condition [0x00007f53ff7fa000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x000000008884ab70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.1/LinkedBlockingQueue.java:433) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:496) "SensorExpiryThread" #25 daemon prio=5 os_prio=0 cpu=155.86ms elapsed=84624.11s tid=0x00007f54991d4800 nid=0x3553 waiting on condition [0x00007f53ff2f9000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088876be0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "ThrottledChannelReaper-Fetch" #26 prio=5 os_prio=0 cpu=2020.83ms elapsed=84624.10s tid=0x00007f54991f4000 nid=0x3554 waiting on condition [0x00007f53ff1f8000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088834790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ThrottledChannelReaper-Produce" #27 prio=5 os_prio=0 cpu=1977.16ms elapsed=84624.09s tid=0x00007f54991f6000 nid=0x3555 waiting on condition [0x00007f53ff0f7000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088876e10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ThrottledChannelReaper-Request" #28 prio=5 os_prio=0 cpu=2039.85ms elapsed=84624.09s tid=0x00007f54991f8800 nid=0x3556 waiting on condition [0x00007f53feff6000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x000000008884ad70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-scheduler-0" #30 daemon prio=5 os_prio=0 cpu=12271.60ms elapsed=84622.76s tid=0x00007f53d4052000 nid=0x3563 waiting on condition [0x00007f54742c5000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089557d40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.1/AbstractQueuedSynchronizer.java:917) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.1/AbstractQueuedSynchronizer.java:1240) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:959) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.cluster.Partition.maybeShrinkIsr(Partition.scala:632) at kafka.server.ReplicaManager$$anonfun$kafka$server$ReplicaManager$$maybeShrinkIsr$2.apply(ReplicaManager.scala:1338) at kafka.server.ReplicaManager$$anonfun$kafka$server$ReplicaManager$$maybeShrinkIsr$2.apply(ReplicaManager.scala:1338) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$maybeShrinkIsr(ReplicaManager.scala:1338) at kafka.server.ReplicaManager$$anonfun$2.apply$mcV$sp(ReplicaManager.scala:323) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.1/Executors.java:515) at java.util.concurrent.FutureTask.runAndReset(java.base@11.0.1/FutureTask.java:305) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@11.0.1/ScheduledThreadPoolExecutor.java:305) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-1" #31 daemon prio=5 os_prio=0 cpu=10906.54ms elapsed=84622.72s tid=0x00007f53d4053000 nid=0x3564 waiting on condition [0x00007f53fc3f4000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-2" #32 daemon prio=5 os_prio=0 cpu=13145.57ms elapsed=84622.71s tid=0x00007f53d4055800 nid=0x3565 waiting on condition [0x00007f53fc2f3000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-3" #33 daemon prio=5 os_prio=0 cpu=12405.26ms elapsed=84622.68s tid=0x00007f53d4058800 nid=0x3566 waiting on condition [0x00007f53fc1f2000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-4" #34 daemon prio=5 os_prio=0 cpu=11033.00ms elapsed=84622.66s tid=0x00007f53d4057800 nid=0x3567 waiting on condition [0x00007f53cc3fc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-5" #35 daemon prio=5 os_prio=0 cpu=12434.72ms elapsed=84622.65s tid=0x00007f53d405c800 nid=0x3568 waiting on condition [0x00007f53b43fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-6" #36 daemon prio=5 os_prio=0 cpu=13301.15ms elapsed=84622.63s tid=0x00007f53d405a800 nid=0x3569 waiting on condition [0x00007f53ac3fc000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-7" #37 daemon prio=5 os_prio=0 cpu=9846.28ms elapsed=84622.62s tid=0x00007f53d405f000 nid=0x356a waiting on condition [0x00007f53cc2fb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-8" #38 daemon prio=5 os_prio=0 cpu=11984.48ms elapsed=84622.16s tid=0x00007f53d4065000 nid=0x356b waiting on condition [0x00007f53997fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-scheduler-9" #39 daemon prio=5 os_prio=0 cpu=12507.09ms elapsed=84622.14s tid=0x00007f53d4066000 nid=0x356c waiting on condition [0x00007f5395afb000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888347d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1177) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-log-cleaner-thread-0" #40 prio=5 os_prio=0 cpu=63395.86ms elapsed=84609.54s tid=0x00007f54988ec800 nid=0x35a5 waiting on condition [0x00007f53feef5000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x000000008956c950> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:1079) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:1369) at java.util.concurrent.CountDownLatch.await(java.base@11.0.1/CountDownLatch.java:278) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:302) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-36130" #41 prio=5 os_prio=0 cpu=3785.23ms elapsed=84609.33s tid=0x00007f549926c000 nid=0x35b7 runnable [0x00007f53bc177000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000891d3688> (a sun.nio.ch.Util$2) - locked <0x00000000891d3698> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at kafka.network.Acceptor.run(SocketServer.scala:390) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "ExpirationReaper-3-Produce" #42 prio=5 os_prio=0 cpu=9989.54ms elapsed=84609.31s tid=0x00007f549928c800 nid=0x35b8 waiting on condition [0x00007f53b42fd000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891b0e10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-3-Fetch" #43 prio=5 os_prio=0 cpu=337962.76ms elapsed=84609.31s tid=0x00007f5499285800 nid=0x35b9 waiting on condition [0x00007f53b41fc000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891e4af8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-3-DeleteRecords" #44 prio=5 os_prio=0 cpu=9850.91ms elapsed=84609.31s tid=0x00007f5499287800 nid=0x35ba waiting on condition [0x00007f53ac2fb000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891f5f50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "LogDirFailureHandler" #45 prio=5 os_prio=0 cpu=0.23ms elapsed=84609.30s tid=0x00007f54992ad800 nid=0x35bb waiting on condition [0x00007f53ac1fa000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000888a2990> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ArrayBlockingQueue.take(java.base@11.0.1/ArrayBlockingQueue.java:417) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:56) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:200) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "controller-event-thread" #46 prio=5 os_prio=0 cpu=3.94ms elapsed=84609.21s tid=0x00007f54992fb000 nid=0x35bc waiting on condition [0x00007f53a01fe000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891f61c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.1/LinkedBlockingQueue.java:433) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:80) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-3-topic" #47 prio=5 os_prio=0 cpu=9775.32ms elapsed=84609.20s tid=0x00007f5499302000 nid=0x35bd runnable [0x00007f5348b1a000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891d3b70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-3-Heartbeat" #48 prio=5 os_prio=0 cpu=9864.83ms elapsed=84609.20s tid=0x00007f5499308800 nid=0x35be waiting on condition [0x00007f5348a19000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891e4e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:265) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ExpirationReaper-3-Rebalance" #49 prio=5 os_prio=0 cpu=9828.86ms elapsed=84609.20s tid=0x00007f5499309800 nid=0x35bf waiting on condition [0x00007f5348918000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891f6430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.DelayQueue.poll(java.base@11.0.1/DelayQueue.java:274) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "group-metadata-manager-0" #50 daemon prio=5 os_prio=0 cpu=2472.79ms elapsed=84609.18s tid=0x00007f5499316000 nid=0x35c0 waiting on condition [0x00007f5348817000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000897e8fa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.1/AbstractQueuedSynchronizer.java:917) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.1/AbstractQueuedSynchronizer.java:1240) at java.util.concurrent.locks.ReentrantLock.lock(java.base@11.0.1/ReentrantLock.java:267) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.coordinator.group.GroupMetadata.inLock(GroupMetadata.scala:197) at kafka.coordinator.group.GroupMetadataManager$$anonfun$cleanupGroupMetadata$2.apply(GroupMetadataManager.scala:742) at kafka.coordinator.group.GroupMetadataManager$$anonfun$cleanupGroupMetadata$2.apply(GroupMetadataManager.scala:740) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.coordinator.group.GroupMetadataManager.cleanupGroupMetadata(GroupMetadataManager.scala:740) at kafka.coordinator.group.GroupMetadataManager.cleanupGroupMetadata(GroupMetadataManager.scala:724) at kafka.coordinator.group.GroupMetadataManager$$anonfun$1.apply$mcV$sp(GroupMetadataManager.scala:143) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.1/Executors.java:515) at java.util.concurrent.FutureTask.runAndReset(java.base@11.0.1/FutureTask.java:305) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@11.0.1/ScheduledThreadPoolExecutor.java:305) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "transaction-log-manager-0" #53 daemon prio=5 os_prio=0 cpu=80.37ms elapsed=84609.15s tid=0x00007f549933d800 nid=0x35c1 waiting on condition [0x00007f5348716000] java.lang.Thread.State: TIMED_WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891d3dd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.1/LockSupport.java:234) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@11.0.1/AbstractQueuedSynchronizer.java:2123) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1182) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "TxnMarkerSenderThread-3" #52 prio=5 os_prio=0 cpu=239.70ms elapsed=84609.15s tid=0x00007f549933f800 nid=0x35c2 runnable [0x00007f5348615000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000891e50b0> (a sun.nio.ch.Util$2) - locked <0x00000000891e50c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-request-handler-0" #54 daemon prio=5 os_prio=0 cpu=369233.72ms elapsed=84609.13s tid=0x00007f5499354800 nid=0x35c3 waiting on condition [0x00007f5348514000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089557d40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1009) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1324) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:738) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.readRecords(Partition.scala:765) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:898) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:965) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:964) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:964) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:817) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:829) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:686) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-1" #55 daemon prio=5 os_prio=0 cpu=388733.76ms elapsed=84609.13s tid=0x00007f5499356000 nid=0x35c4 waiting on condition [0x00007f5348413000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000893ff5e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.1/AbstractQueuedSynchronizer.java:917) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.1/AbstractQueuedSynchronizer.java:1240) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:959) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.cluster.Partition.maybeExpandIsr(Partition.scala:501) at kafka.cluster.Partition.updateReplicaLogReadResult(Partition.scala:473) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:1359) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:1353) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:1353) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:825) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:829) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:686) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-2" #56 daemon prio=5 os_prio=0 cpu=369427.47ms elapsed=84609.13s tid=0x00007f5499359000 nid=0x35c5 waiting on condition [0x00007f5348312000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000893ff5e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1009) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1324) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:738) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.fetchOffsetSnapshot(Partition.scala:832) at kafka.server.DelayedFetch$$anonfun$tryComplete$1.apply(DelayedFetch.scala:87) at kafka.server.DelayedFetch$$anonfun$tryComplete$1.apply(DelayedFetch.scala:79) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:79) at kafka.server.DelayedOperation.maybeTryComplete(DelayedOperation.scala:121) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:277) at kafka.server.ReplicaManager.tryCompleteDelayedFetch(ReplicaManager.scala:307) at kafka.cluster.Partition$$anonfun$13.apply(Partition.scala:743) at kafka.cluster.Partition$$anonfun$13.apply(Partition.scala:729) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:728) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:735) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:723) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.immutable.Map$Map1.foreach(Map.scala:116) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:469) at kafka.coordinator.group.GroupMetadataManager.appendForGroup(GroupMetadataManager.scala:280) at kafka.coordinator.group.GroupMetadataManager.storeOffsets(GroupMetadataManager.scala:423) at kafka.coordinator.group.GroupCoordinator$$anonfun$doCommitOffsets$1.apply$mcV$sp(GroupCoordinator.scala:518) at kafka.coordinator.group.GroupCoordinator$$anonfun$doCommitOffsets$1.apply(GroupCoordinator.scala:503) at kafka.coordinator.group.GroupCoordinator$$anonfun$doCommitOffsets$1.apply(GroupCoordinator.scala:503) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.coordinator.group.GroupMetadata.inLock(GroupMetadata.scala:197) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:502) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:481) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:360) at kafka.server.KafkaApis.handle(KafkaApis.scala:114) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-3" #57 daemon prio=5 os_prio=0 cpu=370330.86ms elapsed=84609.13s tid=0x00007f549935b000 nid=0x35c6 waiting on condition [0x00007f5344f1c000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089557d40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1009) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1324) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:738) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.fetchOffsetForTimestamp(Partition.scala:807) at kafka.server.ReplicaManager.fetchOffsetForTimestamp(ReplicaManager.scala:780) at kafka.server.KafkaApis$$anonfun$18.apply(KafkaApis.scala:820) at kafka.server.KafkaApis$$anonfun$18.apply(KafkaApis.scala:804) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.KafkaApis.handleListOffsetRequestV1AndAbove(KafkaApis.scala:804) at kafka.server.KafkaApis.handleListOffsetRequest(KafkaApis.scala:744) at kafka.server.KafkaApis.handle(KafkaApis.scala:108) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-4" #58 daemon prio=5 os_prio=0 cpu=370616.02ms elapsed=84609.13s tid=0x00007f549935c800 nid=0x35c7 waiting on condition [0x00007f5344e1b000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089557d40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.1/AbstractQueuedSynchronizer.java:917) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.1/AbstractQueuedSynchronizer.java:1240) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:959) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.cluster.Partition.maybeExpandIsr(Partition.scala:501) at kafka.cluster.Partition.updateReplicaLogReadResult(Partition.scala:473) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:1359) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:1353) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:1353) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:825) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:829) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:686) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-5" #59 daemon prio=5 os_prio=0 cpu=371719.16ms elapsed=84609.13s tid=0x00007f549935e800 nid=0x35c8 waiting on condition [0x00007f5344d1a000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089557d40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1009) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1324) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:738) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.fetchOffsetSnapshot(Partition.scala:832) at kafka.server.DelayedFetch$$anonfun$tryComplete$1.apply(DelayedFetch.scala:87) at kafka.server.DelayedFetch$$anonfun$tryComplete$1.apply(DelayedFetch.scala:79) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:79) at kafka.server.DelayedOperation.maybeTryComplete(DelayedOperation.scala:121) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:277) at kafka.server.ReplicaManager.tryCompleteDelayedFetch(ReplicaManager.scala:307) at kafka.cluster.Partition$$anonfun$13.apply(Partition.scala:743) at kafka.cluster.Partition$$anonfun$13.apply(Partition.scala:729) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:728) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:735) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:723) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:469) at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:475) at kafka.server.KafkaApis.handle(KafkaApis.scala:106) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-6" #60 daemon prio=5 os_prio=0 cpu=370043.76ms elapsed=84609.13s tid=0x00007f5499360800 nid=0x35c9 waiting on condition [0x00007f5344c19000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000893ff5e8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1009) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1324) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:738) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.fetchOffsetSnapshot(Partition.scala:832) at kafka.server.DelayedFetch$$anonfun$tryComplete$1.apply(DelayedFetch.scala:87) at kafka.server.DelayedFetch$$anonfun$tryComplete$1.apply(DelayedFetch.scala:79) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:79) at kafka.server.DelayedOperation.maybeTryComplete(DelayedOperation.scala:121) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:277) at kafka.server.ReplicaManager.tryCompleteDelayedFetch(ReplicaManager.scala:307) at kafka.cluster.Partition$$anonfun$13.apply(Partition.scala:743) at kafka.cluster.Partition$$anonfun$13.apply(Partition.scala:729) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:728) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:735) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:723) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:723) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:469) at kafka.server.KafkaApis.handleProduceRequest(KafkaApis.scala:475) at kafka.server.KafkaApis.handle(KafkaApis.scala:106) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-request-handler-7" #61 daemon prio=5 os_prio=0 cpu=370145.10ms elapsed=84609.13s tid=0x00007f5499362000 nid=0x35ca waiting on condition [0x00007f5344b18000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000897e8fa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.1/AbstractQueuedSynchronizer.java:917) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.1/AbstractQueuedSynchronizer.java:1240) at java.util.concurrent.locks.ReentrantLock.lock(java.base@11.0.1/ReentrantLock.java:267) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.coordinator.group.GroupMetadata.inLock(GroupMetadata.scala:197) at kafka.coordinator.group.GroupMetadataManager.getOffsets(GroupMetadataManager.scala:449) at kafka.coordinator.group.GroupCoordinator.handleFetchOffsets(GroupCoordinator.scala:531) at kafka.server.KafkaApis.kafka$server$KafkaApis$$createResponse$2(KafkaApis.scala:1067) at kafka.server.KafkaApis$$anonfun$handleOffsetFetchRequest$1.apply(KafkaApis.scala:1094) at kafka.server.KafkaApis$$anonfun$handleOffsetFetchRequest$1.apply(KafkaApis.scala:1094) at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2246) at kafka.server.KafkaApis.handleOffsetFetchRequest(KafkaApis.scala:1094) at kafka.server.KafkaApis.handle(KafkaApis.scala:115) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:69) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "/config/changes-event-process-thread" #62 prio=5 os_prio=0 cpu=126.07ms elapsed=84609.12s tid=0x00007f549936f800 nid=0x35cc waiting on condition [0x00007f5344a17000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000891b1e38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.1/LinkedBlockingQueue.java:433) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "kafka-network-thread-3-ListenerName(PLAINTEXT)-PLAINTEXT-0" #63 prio=5 os_prio=0 cpu=1852089.42ms elapsed=84609.04s tid=0x00007f5499371000 nid=0x35cf runnable [0x00007f5344716000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000891c2650> (a sun.nio.ch.Util$2) - locked <0x00000000891c2660> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-network-thread-3-ListenerName(PLAINTEXT)-PLAINTEXT-1" #64 prio=5 os_prio=0 cpu=1466361.79ms elapsed=84609.04s tid=0x00007f5499387000 nid=0x35d0 runnable [0x00007f532651e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000891c2928> (a sun.nio.ch.Util$2) - locked <0x00000000891c2938> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "kafka-network-thread-3-ListenerName(PLAINTEXT)-PLAINTEXT-2" #65 prio=5 os_prio=0 cpu=1864403.58ms elapsed=84609.04s tid=0x00007f5499389000 nid=0x35d1 runnable [0x00007f532641d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000891edf30> (a sun.nio.ch.Util$2) - locked <0x00000000891edf40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at kafka.network.Processor.poll(SocketServer.scala:689) at kafka.network.Processor.run(SocketServer.scala:594) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "DestroyJavaVM" #67 prio=5 os_prio=0 cpu=2482.71ms elapsed=84609.03s tid=0x00007f5498011800 nid=0x352c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "ReplicaFetcherThread-1-2" #68 prio=5 os_prio=0 cpu=44260.75ms elapsed=84608.62s tid=0x00007f541402d000 nid=0x35d3 runnable [0x00007f532631c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000891f59a8> (a sun.nio.ch.Util$2) - locked <0x00000000891f5448> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:99) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:199) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-1-1" #69 prio=5 os_prio=0 cpu=174688.86ms elapsed=84608.62s tid=0x00007f5414034000 nid=0x35d4 runnable [0x00007f532621b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000892a2698> (a sun.nio.ch.Util$2) - locked <0x00000000892a26a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:99) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:199) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-0-2" #70 prio=5 os_prio=0 cpu=110295.61ms elapsed=84608.61s tid=0x00007f541403c800 nid=0x35d5 runnable [0x00007f532611a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000892b84a8> (a sun.nio.ch.Util$2) - locked <0x00000000892b84b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:99) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:199) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "ReplicaFetcherThread-0-1" #71 prio=5 os_prio=0 cpu=170842.13ms elapsed=84608.60s tid=0x00007f5414041800 nid=0x35d6 runnable [0x00007f5326019000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPoll.wait(java.base@11.0.1/Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@11.0.1/EPollSelectorImpl.java:120) at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@11.0.1/SelectorImpl.java:124) - locked <0x00000000892e2078> (a sun.nio.ch.Util$2) - locked <0x00000000892e2088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(java.base@11.0.1/SelectorImpl.java:136) at org.apache.kafka.common.network.Selector.select(Selector.java:752) at org.apache.kafka.common.network.Selector.poll(Selector.java:451) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:535) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:96) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:99) at kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:199) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$processFetchRequest(AbstractFetcherThread.scala:241) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread$$anonfun$maybeFetch$1.apply(AbstractFetcherThread.scala:129) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) "executor-Fetch" #72 prio=5 os_prio=0 cpu=834060.94ms elapsed=84607.56s tid=0x00007f53c8004000 nid=0x35e1 waiting on condition [0x00007f5325318000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089557d40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1009) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(java.base@11.0.1/AbstractQueuedSynchronizer.java:1324) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(java.base@11.0.1/ReentrantReadWriteLock.java:738) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.readRecords(Partition.scala:765) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:898) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:965) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:964) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:964) at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:153) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:70) at kafka.server.DelayedOperation.run(DelayedOperation.scala:143) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.1/Executors.java:515) at java.util.concurrent.FutureTask.run(java.base@11.0.1/FutureTask.java:264) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "executor-Produce" #73 prio=5 os_prio=0 cpu=12.75ms elapsed=84307.90s tid=0x00007f53c0006800 nid=0x6458 waiting on condition [0x00007f53cc1fa000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000894ffda0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.1/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "executor-Rebalance" #74 prio=5 os_prio=0 cpu=12.65ms elapsed=83676.55s tid=0x00007f53e0002800 nid=0x64b6 waiting on condition [0x00007f53808d9000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000089534f88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.LinkedBlockingQueue.take(java.base@11.0.1/LinkedBlockingQueue.java:433) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-2" #75 daemon prio=5 os_prio=0 cpu=10.38ms elapsed=83209.92s tid=0x00007f542400a000 nid=0x4a58 waiting on condition [0x00007f53807d8000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-3" #76 daemon prio=5 os_prio=0 cpu=22.20ms elapsed=83209.93s tid=0x00007f5424010000 nid=0x4a59 waiting on condition [0x00007f53806d7000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "executor-Heartbeat" #77 prio=5 os_prio=0 cpu=15.84ms elapsed=82965.35s tid=0x00007f53e4001000 nid=0x7235 waiting on condition [0x00007f5349f1c000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x00000000897e8fa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.1/AbstractQueuedSynchronizer.java:885) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.1/AbstractQueuedSynchronizer.java:917) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.1/AbstractQueuedSynchronizer.java:1240) at java.util.concurrent.locks.ReentrantLock.lock(java.base@11.0.1/ReentrantLock.java:267) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.coordinator.group.GroupMetadata.inLock(GroupMetadata.scala:197) at kafka.coordinator.group.GroupCoordinator.onExpireHeartbeat(GroupCoordinator.scala:833) at kafka.coordinator.group.DelayedHeartbeat.onExpiration(DelayedHeartbeat.scala:34) at kafka.server.DelayedOperation.run(DelayedOperation.scala:144) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.1/Executors.java:515) at java.util.concurrent.FutureTask.run(java.base@11.0.1/FutureTask.java:264) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1128) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-4" #78 daemon prio=5 os_prio=0 cpu=13.47ms elapsed=79608.95s tid=0x00007f5424019000 nid=0x7fea waiting on condition [0x00007f5349e1b000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-5" #79 daemon prio=5 os_prio=0 cpu=6.20ms elapsed=79608.95s tid=0x00007f5424006000 nid=0x7feb waiting on condition [0x00007f5349d1a000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-6" #80 daemon prio=5 os_prio=0 cpu=15.54ms elapsed=76027.10s tid=0x00007f542401b800 nid=0x1f7a waiting on condition [0x00007f5349c19000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-7" #81 daemon prio=5 os_prio=0 cpu=10.59ms elapsed=76027.10s tid=0x00007f5424028000 nid=0x1f7b waiting on condition [0x00007f5349b18000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "logback-8" #82 daemon prio=5 os_prio=0 cpu=18.09ms elapsed=72417.03s tid=0x00007f5424029800 nid=0x624b waiting on condition [0x00007f5349a17000] java.lang.Thread.State: WAITING (parking) at jdk.internal.misc.Unsafe.park(java.base@11.0.1/Native Method) - parking to wait for <0x0000000088402268> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(java.base@11.0.1/LockSupport.java:194) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@11.0.1/AbstractQueuedSynchronizer.java:2081) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:1170) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@11.0.1/ScheduledThreadPoolExecutor.java:899) at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@11.0.1/ThreadPoolExecutor.java:1054) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.1/ThreadPoolExecutor.java:1114) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.1/ThreadPoolExecutor.java:628) at java.lang.Thread.run(java.base@11.0.1/Thread.java:834) "Attach Listener" #83 daemon prio=9 os_prio=0 cpu=0.32ms elapsed=0.10s tid=0x00007f5458001800 nid=0x4a84 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "VM Thread" os_prio=0 cpu=8951.89ms elapsed=84626.05s tid=0x00007f54987ea000 nid=0x3532 runnable "GC Thread#0" os_prio=0 cpu=30655.98ms elapsed=84626.10s tid=0x00007f5498039000 nid=0x352d runnable "GC Thread#1" os_prio=0 cpu=30488.07ms elapsed=84624.97s tid=0x00007f545c001000 nid=0x3544 runnable "GC Thread#2" os_prio=0 cpu=30489.53ms elapsed=84624.97s tid=0x00007f545c002800 nid=0x3545 runnable "GC Thread#3" os_prio=0 cpu=30569.81ms elapsed=84624.97s tid=0x00007f545c004000 nid=0x3546 runnable "GC Thread#4" os_prio=0 cpu=30539.83ms elapsed=84624.97s tid=0x00007f545c005800 nid=0x3547 runnable "GC Thread#5" os_prio=0 cpu=30508.28ms elapsed=84624.97s tid=0x00007f545c007800 nid=0x3548 runnable "G1 Main Marker" os_prio=0 cpu=6.17ms elapsed=84626.10s tid=0x00007f549806c000 nid=0x352e runnable "G1 Conc#0" os_prio=0 cpu=207.80ms elapsed=84626.10s tid=0x00007f549806e000 nid=0x352f runnable "G1 Conc#1" os_prio=0 cpu=203.94ms elapsed=84624.96s tid=0x00007f5470001000 nid=0x3549 runnable "G1 Refine#0" os_prio=0 cpu=4.83ms elapsed=84626.07s tid=0x00007f5498720800 nid=0x3530 runnable "G1 Young RemSet Sampling" os_prio=0 cpu=52437.61ms elapsed=84626.07s tid=0x00007f5498722800 nid=0x3531 runnable "VM Periodic Task Thread" os_prio=0 cpu=35241.06ms elapsed=84625.55s tid=0x00007f5498b36800 nid=0x3541 waiting on condition JNI global refs: 21, weak refs: 0