2017-09-25 07:38:06 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.131-b11 mixed mode): "Attach Listener" #1643 daemon prio=9 os_prio=0 tid=0x00007fd1d0001000 nid=0x1de7 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Agent DNS Service 1498" #1642 daemon prio=5 os_prio=0 tid=0x00007fd0f800c000 nid=0x1bea waiting on condition [0x00007fd050833000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c00b0a20> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "executor-Produce" #1641 prio=5 os_prio=0 tid=0x00007fcfe0010800 nid=0x1be0 waiting for monitor entry [0x00007fd08c1c2000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupMetadataManager.kafka$coordinator$group$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:313) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupMetadataManager$$anonfun$storeOffsets$3.apply(GroupMetadataManager.scala:381) at kafka.coordinator.group.GroupMetadataManager$$anonfun$storeOffsets$3.apply(GroupMetadataManager.scala:381) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:131) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:66) at kafka.server.DelayedOperation.run(DelayedOperation.scala:112) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x000000072ea00c90> (a java.util.concurrent.ThreadPoolExecutor$Worker) "Agent DNS Service 1497" #1640 daemon prio=5 os_prio=0 tid=0x00007fd100019000 nid=0x1420 waiting on condition [0x00007fd03ca35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c00b0a20> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Agent DNS Service 1495" #1638 daemon prio=5 os_prio=0 tid=0x00007fd13001a000 nid=0x141c waiting on condition [0x00007fd1936f7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c00b0a20> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Agent DNS Service 1493" #1636 daemon prio=5 os_prio=0 tid=0x00007fd14002b800 nid=0x126e waiting on condition [0x00007fd03c934000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c00b0a20> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Connection(25)-10.1.10.165" #1156 daemon prio=5 os_prio=0 tid=0x00007fd184005000 nid=0x477 runnable [0x00007fd04a92f000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at java.io.BufferedInputStream.fill(BufferedInputStream.java:246) at java.io.BufferedInputStream.read(BufferedInputStream.java:265) - locked <0x0000000515f42a80> (a java.io.BufferedInputStream) at java.io.FilterInputStream.read(FilterInputStream.java:83) at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$0(TCPTransport.java:683) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$7/728810190.run(Unknown Source) at java.security.AccessController.doPrivileged(Native Method) at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00000005093f0588> (a java.util.concurrent.ThreadPoolExecutor$Worker) "executor-Heartbeat" #702 prio=5 os_prio=0 tid=0x00007fcfc423f800 nid=0x4c1b waiting for monitor entry [0x00007fd03c833000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupCoordinator.onExpireHeartbeat(GroupCoordinator.scala:777) - waiting to lock <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.DelayedHeartbeat.onExpiration(DelayedHeartbeat.scala:38) at kafka.server.DelayedOperation.run(DelayedOperation.scala:113) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x0000000503bc4bd8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "executor-Rebalance" #124 prio=5 os_prio=0 tid=0x00007fcfc8001000 nid=0x22f8 waiting on condition [0x00007fd04c233000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6972f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "JMX server connection timeout 123" #123 daemon prio=5 os_prio=0 tid=0x00007fd150022000 nid=0x22f7 in Object.wait() [0x00007fd04c334000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168) - locked <0x00000004ef1d4e40> (a [I) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI Scheduler(0)" #121 daemon prio=5 os_prio=0 tid=0x00007fd148012000 nid=0x22f5 waiting on condition [0x00007fd04c536000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0845d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "executor-Fetch" #119 prio=5 os_prio=0 tid=0x00007fcfd401d800 nid=0x22f3 waiting on condition [0x00007fd072f52000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a0d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-102" #118 prio=5 os_prio=0 tid=0x00007fcfc4233800 nid=0x22f2 runnable [0x00007fd073253000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed34d9a8> (a sun.nio.ch.Util$3) - locked <0x00000004ed34d9b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed34d960> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-102" #117 prio=5 os_prio=0 tid=0x00007fcfc4218800 nid=0x22f1 runnable [0x00007fd073354000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed47bd10> (a sun.nio.ch.Util$3) - locked <0x00000004ed47bd20> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed47bcc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-105" #116 prio=5 os_prio=0 tid=0x00007fcfc41fe000 nid=0x22f0 runnable [0x00007fd073455000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed5edf40> (a sun.nio.ch.Util$3) - locked <0x00000004ed5edf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed5edef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-105" #115 prio=5 os_prio=0 tid=0x00007fcfc41e3000 nid=0x22ef runnable [0x00007fd07a50e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed400ac0> (a sun.nio.ch.Util$3) - locked <0x00000004ed400ad0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed400a78> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-105" #114 prio=5 os_prio=0 tid=0x00007fcfc41c8000 nid=0x22ee runnable [0x00007fd07a60f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed161610> (a sun.nio.ch.Util$3) - locked <0x00000004ed161620> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed1615c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-102" #113 prio=5 os_prio=0 tid=0x00007fcfc41ad000 nid=0x22ed runnable [0x00007fd07a710000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed5ee2a8> (a sun.nio.ch.Util$3) - locked <0x00000004ed5ee2b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed5ee260> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1" #112 prio=5 os_prio=0 tid=0x00007fcfc4192000 nid=0x22ec runnable [0x00007fd07a811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed400e28> (a sun.nio.ch.Util$3) - locked <0x00000004ed400e38> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed400de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-102" #111 prio=5 os_prio=0 tid=0x00007fcfc4177800 nid=0x22eb runnable [0x00007fd07a912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed47c078> (a sun.nio.ch.Util$3) - locked <0x00000004ed47c088> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed47c030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1" #110 prio=5 os_prio=0 tid=0x00007fcfc415c800 nid=0x22ea runnable [0x00007fd07aa13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed256ab0> (a sun.nio.ch.Util$3) - locked <0x00000004ed256ac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed256a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-101" #109 prio=5 os_prio=0 tid=0x00007fcfc4141800 nid=0x22e9 runnable [0x00007fd07ab14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed401190> (a sun.nio.ch.Util$3) - locked <0x00000004ed4011a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed401148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-101" #108 prio=5 os_prio=0 tid=0x00007fcfc4127000 nid=0x22e8 runnable [0x00007fd07ac15000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed5ee610> (a sun.nio.ch.Util$3) - locked <0x00000004ed5ee620> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed5ee5c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1" #107 prio=5 os_prio=0 tid=0x00007fcfc410c000 nid=0x22e7 runnable [0x00007fd07ad16000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed47c3e0> (a sun.nio.ch.Util$3) - locked <0x00000004ed47c3f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed47c398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-101" #106 prio=5 os_prio=0 tid=0x00007fcfc40f1000 nid=0x22e6 runnable [0x00007fd08bebf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed256e18> (a sun.nio.ch.Util$3) - locked <0x00000004ed256e28> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed256dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-101" #105 prio=5 os_prio=0 tid=0x00007fcfc40d7000 nid=0x22e5 runnable [0x00007fd08bfc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed4014f8> (a sun.nio.ch.Util$3) - locked <0x00000004ed401508> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ed4014b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-101" #101 prio=5 os_prio=0 tid=0x00007fcfc40bd000 nid=0x22e4 runnable [0x00007fd08c0c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6a31a0> (a sun.nio.ch.Util$3) - locked <0x00000004ec6a31b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6a3158> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-102" #100 prio=5 os_prio=0 tid=0x00007fcfc40a0000 nid=0x22e0 runnable [0x00007fd050a35000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec696550> (a sun.nio.ch.Util$3) - locked <0x00000004ec696560> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec696508> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-105" #99 prio=5 os_prio=0 tid=0x00007fcfc4085000 nid=0x22df runnable [0x00007fd050b36000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec695658> (a sun.nio.ch.Util$3) - locked <0x00000004ec695668> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec695610> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-102" #98 prio=5 os_prio=0 tid=0x00007fcfc406a800 nid=0x22de runnable [0x00007fd059930000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6991a0> (a sun.nio.ch.Util$3) - locked <0x00000004ec6991b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec699158> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1" #97 prio=5 os_prio=0 tid=0x00007fcfc4036800 nid=0x22dd runnable [0x00007fd059a31000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec697628> (a sun.nio.ch.Util$3) - locked <0x00000004ec697638> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6975e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-105" #96 prio=5 os_prio=0 tid=0x00007fcfc401a000 nid=0x22dc runnable [0x00007fd059b32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec694988> (a sun.nio.ch.Util$3) - locked <0x00000004ec694998> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec694940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:89) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:93) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:207) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:42) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:151) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:112) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "kafka-request-handler-7" #95 daemon prio=5 os_prio=0 tid=0x00007fd282250000 nid=0x22db waiting for monitor entry [0x00007fd059e33000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.DelayedProduce.safeTryComplete(DelayedProduce.scala:75) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:338) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:244) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:250) at kafka.cluster.Partition.tryCompleteDelayedRequests(Partition.scala:418) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:500) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:546) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:532) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.immutable.Map$Map1.foreach(Map.scala:116) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:532) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:373) at kafka.coordinator.group.GroupMetadataManager.appendForGroup(GroupMetadataManager.scala:239) at kafka.coordinator.group.GroupMetadataManager.storeOffsets(GroupMetadataManager.scala:381) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:465) - locked <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #94 daemon prio=5 os_prio=0 tid=0x00007fd28224e000 nid=0x22da waiting for monitor entry [0x00007fd059f34000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:449) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #93 daemon prio=5 os_prio=0 tid=0x00007fd28224c000 nid=0x22d9 waiting for monitor entry [0x00007fd05a035000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.DelayedProduce.safeTryComplete(DelayedProduce.scala:75) - waiting to lock <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:338) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:244) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:250) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:1095) at kafka.server.ReplicaManager$$anonfun$updateFollowerLogReadResults$2.apply(ReplicaManager.scala:1088) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.updateFollowerLogReadResults(ReplicaManager.scala:1088) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:622) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:606) at kafka.server.KafkaApis.handle(KafkaApis.scala:98) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #92 daemon prio=5 os_prio=0 tid=0x00007fd28224a800 nid=0x22d8 waiting for monitor entry [0x00007fd05a136000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:449) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #91 daemon prio=5 os_prio=0 tid=0x00007fd282248800 nid=0x22d7 waiting for monitor entry [0x00007fd05cb26000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:449) - waiting to lock <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #90 daemon prio=5 os_prio=0 tid=0x00007fd28228b000 nid=0x22d6 waiting for monitor entry [0x00007fd05cc27000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:449) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #89 daemon prio=5 os_prio=0 tid=0x00007fd282289800 nid=0x22d5 waiting for monitor entry [0x00007fd05cd27000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.DelayedProduce.safeTryComplete(DelayedProduce.scala:75) - waiting to lock <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:338) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:244) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:250) at kafka.cluster.Partition.tryCompleteDelayedRequests(Partition.scala:418) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:500) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:546) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:532) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.immutable.Map$Map1.foreach(Map.scala:116) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:532) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:373) at kafka.coordinator.group.GroupMetadataManager.appendForGroup(GroupMetadataManager.scala:239) at kafka.coordinator.group.GroupMetadataManager.storeOffsets(GroupMetadataManager.scala:381) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:465) - locked <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #88 daemon prio=5 os_prio=0 tid=0x00007fd282288000 nid=0x22d4 waiting for monitor entry [0x00007fd05ce29000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:449) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #87 daemon prio=5 os_prio=0 tid=0x00007fd281bb8000 nid=0x22d3 waiting on condition [0x00007fd05cf2a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a2560> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) Locked ownable synchronizers: - None "main-SendThread(kafka-chq-txn104.chq.ei:2181)" #86 daemon prio=5 os_prio=0 tid=0x00007fd282611000 nid=0x22d2 runnable [0x00007fd05d02b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec694cb8> (a sun.nio.ch.Util$3) - locked <0x00000004ec694cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec694c70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) Locked ownable synchronizers: - None "ZkClient-EventThread-85-kafka-chq-txn101.chq.ei:2181,kafka-chq-txn102.chq.ei:2181,kafka-chq-txn103.chq.ei:2181,kafka-chq-txn104.chq.ei:2181,kafka-chq-txn105.chq.ei:2181/cp01" #85 daemon prio=5 os_prio=0 tid=0x00007fd1c4880800 nid=0x22d1 waiting on condition [0x00007fd05d12c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6995c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:68) Locked ownable synchronizers: - None "TxnMarkerSenderThread-103" #83 prio=5 os_prio=0 tid=0x00007fd28245d000 nid=0x22d0 runnable [0x00007fd05d22d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6a11b8> (a sun.nio.ch.Util$3) - locked <0x00000004ec6a11c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6a1170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:433) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:73) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "transaction-log-manager-0" #84 daemon prio=5 os_prio=0 tid=0x00007fd282633000 nid=0x22cf waiting on condition [0x00007fd05d32e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a2760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #81 daemon prio=5 os_prio=0 tid=0x00007fd281db6800 nid=0x22ce waiting on condition [0x00007fd05d42f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a3560> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-103-Rebalance" #80 prio=5 os_prio=0 tid=0x00007fd28275d000 nid=0x22cd runnable [0x00007fd05d730000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec677470> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:383) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:407) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ExpirationReaper-103-Heartbeat" #79 prio=5 os_prio=0 tid=0x00007fd28275c000 nid=0x22cc runnable [0x00007fd05d831000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec69fc50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:383) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:407) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ExpirationReaper-103-topic" #78 prio=5 os_prio=0 tid=0x00007fd1c497c800 nid=0x22cb runnable [0x00007fd05d932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a1280> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:383) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:407) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "controller-event-thread" #77 prio=5 os_prio=0 tid=0x00007fd283dd6000 nid=0x22ca waiting on condition [0x00007fd05da33000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a3628> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:45) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ExpirationReaper-103-DeleteRecords" #76 prio=5 os_prio=0 tid=0x00007fd283b01000 nid=0x22c9 runnable [0x00007fd05db34000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec69e488> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:383) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:407) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ExpirationReaper-103-Fetch" #75 prio=5 os_prio=0 tid=0x00007fd1c457f800 nid=0x22c8 runnable [0x00007fd05dc35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a9e10> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:383) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:407) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ExpirationReaper-103-Produce" #74 prio=5 os_prio=0 tid=0x00007fd2830b7800 nid=0x22c7 runnable [0x00007fd05dd36000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6a29c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:383) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:407) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SSL)-SSL-9093" #73 prio=5 os_prio=0 tid=0x00007fd282637000 nid=0x22c6 runnable [0x00007fd0677da000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6ab8d8> (a sun.nio.ch.Util$3) - locked <0x00000004ec6ab8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6a36f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:271) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-103-ListenerName(SSL)-SSL-5" #72 prio=5 os_prio=0 tid=0x00007fd283dee000 nid=0x22c5 runnable [0x00007fd0678db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ece4f8e8> (a sun.nio.ch.Util$3) - locked <0x00000004ece4f908> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec697888> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at kafka.network.Processor.poll(SocketServer.scala:500) at kafka.network.Processor.run(SocketServer.scala:435) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-103-ListenerName(SSL)-SSL-4" #71 prio=5 os_prio=0 tid=0x00007fd2829e7000 nid=0x22c4 runnable [0x00007fd0679dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ed08e2e8> (a sun.nio.ch.Util$3) - locked <0x00000004ed08e328> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ecfb6d20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at kafka.network.Processor.poll(SocketServer.scala:500) at kafka.network.Processor.run(SocketServer.scala:435) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-103-ListenerName(SSL)-SSL-3" #70 prio=5 os_prio=0 tid=0x00007fd281e5b800 nid=0x22c3 runnable [0x00007fd08b953000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6aab38> (a sun.nio.ch.Util$3) - locked <0x00000004ec6aab48> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6a2ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at kafka.network.Processor.poll(SocketServer.scala:500) at kafka.network.Processor.run(SocketServer.scala:435) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-9092" #69 prio=5 os_prio=0 tid=0x00007fd1c4494800 nid=0x223a runnable [0x00007fd08bc9b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec677738> (a sun.nio.ch.Util$3) - locked <0x00000004ec677748> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6776f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:271) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-103-ListenerName(PLAINTEXT)-PLAINTEXT-2" #68 prio=5 os_prio=0 tid=0x00007fd1c4f10000 nid=0x2239 runnable [0x00007fd08bd9c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6abb50> (a sun.nio.ch.Util$3) - locked <0x00000004ec6abb60> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec6abb08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at kafka.network.Processor.poll(SocketServer.scala:500) at kafka.network.Processor.run(SocketServer.scala:435) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-103-ListenerName(PLAINTEXT)-PLAINTEXT-1" #67 prio=5 os_prio=0 tid=0x00007fd1c44af000 nid=0x2238 runnable [0x00007fd191e20000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec6779b0> (a sun.nio.ch.Util$3) - locked <0x00000004ec6779c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec677968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at kafka.network.Processor.poll(SocketServer.scala:500) at kafka.network.Processor.run(SocketServer.scala:435) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-103-ListenerName(PLAINTEXT)-PLAINTEXT-0" #66 prio=5 os_prio=0 tid=0x00007fd1c442d000 nid=0x2237 runnable [0x00007fd1926ed000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004ec69d508> (a sun.nio.ch.Util$3) - locked <0x00000004ec69d518> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004ec69d4c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:529) at org.apache.kafka.common.network.Selector.poll(Selector.java:321) at kafka.network.Processor.poll(SocketServer.scala:500) at kafka.network.Processor.run(SocketServer.scala:435) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #65 prio=5 os_prio=0 tid=0x00007fd1c4e71000 nid=0x2236 waiting on condition [0x00007fd191f21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec69fec0> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:276) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:236) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "Timer-0" #64 daemon prio=5 os_prio=0 tid=0x00007fd1ac1df000 nid=0x222f in Object.wait() [0x00007fd104134000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000004c130f330> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) Locked ownable synchronizers: - None "Temp Instrumentation Queue Heartbeat" #63 daemon prio=5 os_prio=0 tid=0x00007fd1b4061000 nid=0x222e in Object.wait() [0x00007fd12e7e5000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004c12c3a20> (a com.wily.util.heartbeat.IntervalHeartbeat) at java.lang.Object.wait(Object.java:502) at com.wily.util.heartbeat.IntervalHeartbeat.waitForBehaviorIfEmpty(IntervalHeartbeat.java:570) - locked <0x00000004c12c3a20> (a com.wily.util.heartbeat.IntervalHeartbeat) at com.wily.util.heartbeat.IntervalHeartbeat.access$1(IntervalHeartbeat.java:562) at com.wily.util.heartbeat.IntervalHeartbeat$HeartbeatRunnable.run(IntervalHeartbeat.java:667) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-9" #62 daemon prio=5 os_prio=0 tid=0x00007fd12802a000 nid=0x222d waiting on condition [0x00007fd1796bd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #61 daemon prio=5 os_prio=0 tid=0x00007fd128024800 nid=0x222c waiting on condition [0x00007fd0f39fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #60 daemon prio=5 os_prio=0 tid=0x00007fd128026800 nid=0x222b waiting on condition [0x00007fd114dfe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #59 daemon prio=5 os_prio=0 tid=0x00007fd128018800 nid=0x222a waiting on condition [0x00007fd179899000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #58 daemon prio=5 os_prio=0 tid=0x00007fd130009800 nid=0x2229 waiting on condition [0x00007fd17a46f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-4" #57 daemon prio=5 os_prio=0 tid=0x00007fd130007800 nid=0x2228 runnable [0x00007fd11c3fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #56 daemon prio=5 os_prio=0 tid=0x00007fd130006800 nid=0x2227 waiting on condition [0x00007fd12c3fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #55 daemon prio=5 os_prio=0 tid=0x00007fd130005800 nid=0x2226 waiting on condition [0x00007fd17bb2d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #54 daemon prio=5 os_prio=0 tid=0x00007fd12801d800 nid=0x2225 waiting on condition [0x00007fd17882c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #53 daemon prio=5 os_prio=0 tid=0x00007fd128017000 nid=0x2224 waiting on condition [0x00007fd17892d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c08ebf38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledRequestReaper-Request" #49 prio=5 os_prio=0 tid=0x00007fd1c5701800 nid=0x2220 runnable [0x00007fd192022000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0da0810> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:164) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ThrottledRequestReaper-Produce" #48 prio=5 os_prio=0 tid=0x00007fd1c56fd800 nid=0x221f runnable [0x00007fd192123000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0a8acc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:164) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "ThrottledRequestReaper-Fetch" #47 prio=5 os_prio=0 tid=0x00007fd1c56fb800 nid=0x221e runnable [0x00007fd192224000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0da09f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledRequestReaper.doWork(ClientQuotaManager.scala:164) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:64) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #46 daemon prio=5 os_prio=0 tid=0x00007fd1c56f3000 nid=0x221d waiting on condition [0x00007fd192325000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0da0bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #45 daemon prio=5 os_prio=0 tid=0x00007fd1c5687000 nid=0x221c runnable [0x00007fd1929f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0da0bb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "SensorExpiryThread" #44 daemon prio=5 os_prio=0 tid=0x00007fd1c568b000 nid=0x221b waiting on condition [0x00007fd192af1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0a8ae88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #43 daemon prio=5 os_prio=0 tid=0x00007fd1c5699800 nid=0x221a waiting on condition [0x00007fd1927ee000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0da0fb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) Locked ownable synchronizers: - None "main-SendThread(kafka-chq-txn103.chq.ei:2181)" #42 daemon prio=5 os_prio=0 tid=0x00007fd1c5698000 nid=0x2219 runnable [0x00007fd1928ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00000004c0a8b1a0> (a sun.nio.ch.Util$3) - locked <0x00000004c0a8b1b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000004c0a8b158> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141) Locked ownable synchronizers: - None "ZkClient-EventThread-41-kafka-chq-txn101.chq.ei:2181,kafka-chq-txn102.chq.ei:2181,kafka-chq-txn103.chq.ei:2181,kafka-chq-txn104.chq.ei:2181,kafka-chq-txn105.chq.ei:2181/cp01" #41 daemon prio=5 os_prio=0 tid=0x00007fd1c565e800 nid=0x2218 waiting on condition [0x00007fd192bf2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004c0da11c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.I0Itec.zkclient.ZkEventThread.run(ZkEventThread.java:68) Locked ownable synchronizers: - None "PO:client_main Mailman 3" #35 daemon prio=5 os_prio=0 tid=0x00007fd18c01c000 nid=0x2214 in Object.wait() [0x00007fd1930f3000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004c0da13c0> (a java.lang.Object) at com.wily.EDU.oswego.cs.dl.util.concurrent.BoundedLinkedQueue.poll(BoundedLinkedQueue.java:253) - locked <0x00000004c0da13c0> (a java.lang.Object) at com.wily.util.concurrent.PooledVaryingExecutor.getTask(PooledVaryingExecutor.java:104) at com.wily.EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:727) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "PO:client_main Mailman 2" #34 daemon prio=5 os_prio=0 tid=0x00007fd18c019800 nid=0x2213 in Object.wait() [0x00007fd1931f4000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004c0da13c0> (a java.lang.Object) at com.wily.EDU.oswego.cs.dl.util.concurrent.BoundedLinkedQueue.poll(BoundedLinkedQueue.java:253) - locked <0x00000004c0da13c0> (a java.lang.Object) at com.wily.util.concurrent.PooledVaryingExecutor.getTask(PooledVaryingExecutor.java:104) at com.wily.EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:727) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "PO:client_main Mailman 1" #33 daemon prio=5 os_prio=0 tid=0x00007fd18c018000 nid=0x2212 in Object.wait() [0x00007fd1932f5000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004c0da13c0> (a java.lang.Object) at com.wily.EDU.oswego.cs.dl.util.concurrent.BoundedLinkedQueue.poll(BoundedLinkedQueue.java:253) - locked <0x00000004c0da13c0> (a java.lang.Object) at com.wily.util.concurrent.PooledVaryingExecutor.getTask(PooledVaryingExecutor.java:104) at com.wily.EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:727) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "PO Route Down Executor" #32 daemon prio=5 os_prio=0 tid=0x00007fd18c015000 nid=0x2211 in Object.wait() [0x00007fd1933f6000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.wily.EDU.oswego.cs.dl.util.concurrent.BoundedLinkedQueue.take(BoundedLinkedQueue.java:225) - locked <0x00000004c0da1928> (a java.lang.Object) at com.wily.EDU.oswego.cs.dl.util.concurrent.QueuedExecutor$RunLoop.run(QueuedExecutor.java:82) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "UnknownHub Hub Receive 2" #31 daemon prio=5 os_prio=0 tid=0x00007fd198043000 nid=0x2210 runnable [0x00007fd193ffe000] java.lang.Thread.State: RUNNABLE at java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) at java.net.SocketInputStream.read(SocketInputStream.java:171) at java.net.SocketInputStream.read(SocketInputStream.java:141) at com.wily.util.io.NonSyncBufferedInputStream.fill(NonSyncBufferedInputStream.java:54) at com.wily.util.io.NonSyncBufferedInputStream.read(NonSyncBufferedInputStream.java:79) at java.io.FilterInputStream.read(FilterInputStream.java:83) at com.wily.util.io.EncryptInputStream.read(EncryptInputStream.java:64) at com.wily.org.apache.commons.io.input.ProxyInputStream.read(ProxyInputStream.java:49) at com.wily.org.apache.commons.io.input.CountingInputStream.read(CountingInputStream.java:70) at java.io.DataInputStream.readInt(DataInputStream.java:387) at com.wily.isengard.postofficehub.link.v1.IsengardObjectInputStream.readInt(IsengardObjectInputStream.java:1295) at com.wily.isengard.postofficehub.link.v1.IsengardObjectInputStream.setUpStartObjectGraph(IsengardObjectInputStream.java:460) at com.wily.isengard.postofficehub.link.v1.IsengardObjectInputStream.readObject(IsengardObjectInputStream.java:258) at com.wily.isengard.postofficehub.link.v1.IncomingMessageDeliveryTask.deliverNextMessage(IncomingMessageDeliveryTask.java:76) at com.wily.isengard.postofficehub.link.v1.IncomingRouteConnector.receiveIncomingMessages(IncomingRouteConnector.java:170) at com.wily.isengard.postofficehub.link.v1.IncomingRouteConnector.doTask(IncomingRouteConnector.java:89) at com.wily.isengard.util.thread.AThreadedExecutable.run(AThreadedExecutable.java:192) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "UnknownHub Hub Transmit 2" #30 daemon prio=5 os_prio=0 tid=0x00007fd19805b800 nid=0x220f in Object.wait() [0x00007fd193afb000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at com.wily.EDU.oswego.cs.dl.util.concurrent.BoundedLinkedQueue.poll(BoundedLinkedQueue.java:253) - locked <0x00000004c09740d0> (a java.lang.Object) at com.wily.isengard.postofficehub.link.v1.OutgoingMessageDeliveryTask.deliverNextMessageInternal(OutgoingMessageDeliveryTask.java:126) at com.wily.isengard.postofficehub.link.v1.OutgoingMessageDeliveryTask.deliverNextMessage(OutgoingMessageDeliveryTask.java:86) at com.wily.isengard.postofficehub.link.v1.OutgoingRouteConnector.sendOutgoingMessages(OutgoingRouteConnector.java:151) at com.wily.isengard.postofficehub.link.v1.OutgoingRouteConnector.doTask(OutgoingRouteConnector.java:73) at com.wily.isengard.util.thread.AThreadedExecutable.run(AThreadedExecutable.java:192) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "AsyncAppender-Dispatcher-Thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007fd1c5149800 nid=0x220d in Object.wait() [0x00007fd1937f8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.log4j.AsyncAppender$Dispatcher.run(AsyncAppender.java:548) - locked <0x00000004c0da1c90> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #27 daemon prio=5 os_prio=0 tid=0x00007fd1c5141800 nid=0x220b runnable [0x00007fd1939fa000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-9095" #26 daemon prio=5 os_prio=0 tid=0x00007fd1c513f000 nid=0x220a runnable [0x00007fd193bfc000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #25 daemon prio=5 os_prio=0 tid=0x00007fd1c5125800 nid=0x2209 runnable [0x00007fd193efd000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Agent ServerConnection" #19 daemon prio=5 os_prio=0 tid=0x00007fd1b4039800 nid=0x2204 waiting on condition [0x00007fd1e4341000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.wily.introscope.agent.connection.ConnectionThread.sendData(ConnectionThread.java:504) at com.wily.introscope.agent.connection.ConnectionThread.run(ConnectionThread.java:70) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Command Queue Heartbeat" #18 daemon prio=5 os_prio=0 tid=0x00007fd1b4035800 nid=0x2203 sleeping[0x00007fd1e4442000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.wily.util.heartbeat.IntervalHeartbeat$HeartbeatRunnable.run(IntervalHeartbeat.java:673) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #16 daemon prio=9 os_prio=0 tid=0x00007fd1c4fff800 nid=0x2202 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread3" #15 daemon prio=9 os_prio=0 tid=0x00007fd1c4ffb000 nid=0x2201 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #14 daemon prio=9 os_prio=0 tid=0x00007fd1c4ff9000 nid=0x2200 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #13 daemon prio=9 os_prio=0 tid=0x00007fd1c4ff6800 nid=0x21ff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #12 daemon prio=9 os_prio=0 tid=0x00007fd1c4ff4800 nid=0x21fe waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Agent Execution" #11 daemon prio=5 os_prio=0 tid=0x00007fd1c4ff2000 nid=0x21fd in Object.wait() [0x00007fd1e4c48000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.wily.util.adt.BlockingQueue.interruptableDequeue(BlockingQueue.java:123) - locked <0x00000004c0076e80> (a com.wily.util.adt.BlockingQueue) at com.wily.util.task.AsynchExecutionQueue.doTask(AsynchExecutionQueue.java:200) at com.wily.util.task.ATask$CoreTask.run(ATask.java:132) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread Monitor Heartbeat Heartbeat" #7 daemon prio=5 os_prio=0 tid=0x00007fd1c4ff1000 nid=0x21fc in Object.wait() [0x00007fd1e4d49000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000004c0076f50> (a com.wily.util.heartbeat.IntervalHeartbeat) at java.lang.Object.wait(Object.java:502) at com.wily.util.heartbeat.IntervalHeartbeat.waitForBehaviorIfEmpty(IntervalHeartbeat.java:570) - locked <0x00000004c0076f50> (a com.wily.util.heartbeat.IntervalHeartbeat) at com.wily.util.heartbeat.IntervalHeartbeat.access$1(IntervalHeartbeat.java:562) at com.wily.util.heartbeat.IntervalHeartbeat$HeartbeatRunnable.run(IntervalHeartbeat.java:667) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Agent Heartbeat" #6 daemon prio=5 os_prio=0 tid=0x00007fd1c4f79000 nid=0x21fb sleeping[0x00007fd1e4e4a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.wily.util.heartbeat.IntervalHeartbeat$HeartbeatRunnable.run(IntervalHeartbeat.java:673) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Remove Metric Data Watch Heartbeat Heartbeat" #9 daemon prio=5 os_prio=0 tid=0x00007fd1c4f8c000 nid=0x21fa waiting on condition [0x00007fd1e4f4b000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.wily.util.heartbeat.IntervalHeartbeat$HeartbeatRunnable.run(IntervalHeartbeat.java:673) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Configuration Watch Heartbeat Heartbeat" #8 daemon prio=5 os_prio=0 tid=0x00007fd1c4ac1800 nid=0x21f9 waiting on condition [0x00007fd1e504c000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at com.wily.util.heartbeat.IntervalHeartbeat$HeartbeatRunnable.run(IntervalHeartbeat.java:673) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Transaction Structure Queue Execution" #10 daemon prio=5 os_prio=0 tid=0x00007fd1c4a5e000 nid=0x21f8 in Object.wait() [0x00007fd1e514d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at com.wily.util.adt.BlockingQueue.interruptableDequeue(BlockingQueue.java:123) - locked <0x00000004c0076ef8> (a com.wily.util.adt.BlockingQueue) at com.wily.util.task.AsynchExecutionQueue.doTask(AsynchExecutionQueue.java:200) at com.wily.util.task.ATask$CoreTask.run(ATask.java:132) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007fd28141e000 nid=0x21f1 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007fd28141c800 nid=0x21f0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007fd2813ea800 nid=0x21ef in Object.wait() [0x00007fd1e690b000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143) - locked <0x00000004c0076ee8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007fd2813e6000 nid=0x21ee in Object.wait() [0x00007fd1e6a0c000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:191) - locked <0x00000004c0076ed8> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007fd280010800 nid=0x21d3 waiting on condition [0x00007fd288418000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004ec6aa250> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:641) at io.confluent.support.metrics.SupportedServerStartable.awaitShutdown(SupportedServerStartable.java:147) at io.confluent.support.metrics.SupportedKafka.main(SupportedKafka.java:50) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007fd2813de800 nid=0x21ed runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007fd280028000 nid=0x21d4 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007fd28002a000 nid=0x21d5 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007fd28002c000 nid=0x21d6 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007fd28002d800 nid=0x21d7 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007fd28002f800 nid=0x21d8 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007fd280031800 nid=0x21d9 runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007fd280033000 nid=0x21da runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007fd280035000 nid=0x21db runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007fd280036800 nid=0x21dc runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007fd280038800 nid=0x21dd runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007fd28005e800 nid=0x21e9 runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fd280060800 nid=0x21ea runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fd280062000 nid=0x21eb runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007fd280064000 nid=0x21ec runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007fd28004e000 nid=0x21e8 runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007fd28004c000 nid=0x21e7 runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007fd28004a000 nid=0x21e6 runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007fd280048800 nid=0x21e5 runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007fd280046800 nid=0x21e4 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007fd280044800 nid=0x21e3 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007fd280043000 nid=0x21e2 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007fd280041000 nid=0x21e1 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007fd28003f000 nid=0x21e0 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007fd28003d800 nid=0x21df runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007fd28003b800 nid=0x21de runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007fd1c5142800 nid=0x220c waiting on condition JNI global references: 354 Found one Java-level deadlock: ============================= "executor-Produce": waiting to lock monitor 0x00007fcfd0030958 (object 0x00000004eef46dd0, a kafka.coordinator.group.GroupMetadata), which is held by "kafka-request-handler-1" "kafka-request-handler-1": waiting to lock monitor 0x00007fcfdc035398 (object 0x00000004eef4a4f8, a kafka.coordinator.group.GroupMetadata), which is held by "kafka-request-handler-7" "kafka-request-handler-7": waiting to lock monitor 0x00007fcfd0030958 (object 0x00000004eef46dd0, a kafka.coordinator.group.GroupMetadata), which is held by "kafka-request-handler-1" Java stack information for the threads listed above: =================================================== "executor-Produce": at kafka.coordinator.group.GroupMetadataManager.kafka$coordinator$group$GroupMetadataManager$$putCacheCallback$2(GroupMetadataManager.scala:313) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupMetadataManager$$anonfun$storeOffsets$3.apply(GroupMetadataManager.scala:381) at kafka.coordinator.group.GroupMetadataManager$$anonfun$storeOffsets$3.apply(GroupMetadataManager.scala:381) at kafka.server.DelayedProduce.onComplete(DelayedProduce.scala:131) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:66) at kafka.server.DelayedOperation.run(DelayedOperation.scala:112) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) "kafka-request-handler-1": at kafka.server.DelayedProduce.safeTryComplete(DelayedProduce.scala:75) - waiting to lock <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:338) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:244) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:250) at kafka.cluster.Partition.tryCompleteDelayedRequests(Partition.scala:418) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:500) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:546) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:532) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.immutable.Map$Map1.foreach(Map.scala:116) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:532) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:373) at kafka.coordinator.group.GroupMetadataManager.appendForGroup(GroupMetadataManager.scala:239) at kafka.coordinator.group.GroupMetadataManager.storeOffsets(GroupMetadataManager.scala:381) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:465) - locked <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) "kafka-request-handler-7": at kafka.server.DelayedProduce.safeTryComplete(DelayedProduce.scala:75) - waiting to lock <0x00000004eef46dd0> (a kafka.coordinator.group.GroupMetadata) at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:338) at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:244) at kafka.server.ReplicaManager.tryCompleteDelayedProduce(ReplicaManager.scala:250) at kafka.cluster.Partition.tryCompleteDelayedRequests(Partition.scala:418) at kafka.cluster.Partition.appendRecordsToLeader(Partition.scala:500) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:546) at kafka.server.ReplicaManager$$anonfun$appendToLocalLog$2.apply(ReplicaManager.scala:532) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.immutable.Map$Map1.foreach(Map.scala:116) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.appendToLocalLog(ReplicaManager.scala:532) at kafka.server.ReplicaManager.appendRecords(ReplicaManager.scala:373) at kafka.coordinator.group.GroupMetadataManager.appendForGroup(GroupMetadataManager.scala:239) at kafka.coordinator.group.GroupMetadataManager.storeOffsets(GroupMetadataManager.scala:381) at kafka.coordinator.group.GroupCoordinator.doCommitOffsets(GroupCoordinator.scala:465) - locked <0x00000004eef4a4f8> (a kafka.coordinator.group.GroupMetadata) at kafka.coordinator.group.GroupCoordinator.handleCommitOffsets(GroupCoordinator.scala:428) at kafka.server.KafkaApis.handleOffsetCommitRequest(KafkaApis.scala:356) at kafka.server.KafkaApis.handle(KafkaApis.scala:105) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:66) at java.lang.Thread.run(Thread.java:748) Found 1 deadlock.